Fixed typo error and added examples for tensorflow and mindspore.

examples/federated-learning-surface-defect-detection-aggregation.Dockerfile

@@ -20,6 +20,6 @@ RUN rm -rf /home/plato/.git
 RUN pip install -r /home/plato/requirements.txt
 
 WORKDIR /home/work
-COPY examples/federated_learning/surface_defect_detection_v2  /home/work/
+COPY examples/federated_learning/surface_defect_detection_pytorch  /home/work/
 
 CMD ["/bin/sh", "-c", "ulimit -n 50000; python aggregate.py"]

examples/federated-learning-surface-defect-detection-train.Dockerfile

@@ -23,6 +23,6 @@ RUN rm -rf /home/plato/.git
 RUN pip install -r /home/plato/requirements.txt
 
 WORKDIR /home/work
-COPY examples/federated_learning/surface_defect_detection_v2  /home/work/
+COPY examples/federated_learning/surface_defect_detection_pytorch  /home/work/
 
 ENTRYPOINT ["python", "train.py"]

examples/federated_learning/surface_defect_detection_mindspore/interface.py

@@ -0,0 +1,130 @@
+# Copyright 2021 The KubeEdge Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numpy as np
+
+import mindspore as ms
+import mindspore.nn as nn
+from mindspore.dataset import GeneratorDataset
+import mindspore.dataset.vision.c_transforms as c_vision
+import mindspore.dataset.vision.py_transforms as py_vision
+
+from sedna.algorithms.aggregation import FedAvgV2
+from sedna.algorithms.client_choose import SimpleClientChoose
+from sedna.common.config import Context
+from sedna.core.federated_learning import FederatedLearningV2
+
+# os.environ['BACKEND_TYPE'] = 'KERAS'
+
+simple_chooser = SimpleClientChoose(per_round=2)
+
+# It has been determined that mistnet is required here.
+fedavg = FedAvgV2()
+
+# The function `get_transmitter_from_config()` returns an object instance.
+s3_transmitter = FederatedLearningV2.get_transmitter_from_config()
+
+class SddDataset():
+    def __init__(self, x, y):
+        self.labels = y
+        self.images = x
+        self.index = 0
+
+    def __len__(self):
+        return len(self.images)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        self.index = self.index % len(self.images)
+        x, y = self.images[self.index], self.labels[self.index]
+        self.index = self.index + 1
+        return x, y
+
+
+class myDataset:
+    def __init__(self, trainset=None, testset=None):
+        self.customized = True
+        transform = [
+            c_vision.Resize((128, 128)),
+            py_vision.ToTensor(),
+            py_vision.Normalize((0.5, ), (0.5, ))
+        ]
+        self.trainset = GeneratorDataset(SddDataset(trainset[0], trainset[1]),
+                                         column_names=["image", "label"])
+        self.testset = GeneratorDataset(SddDataset(trainset[0], trainset[1]),
+                                         column_names=["image", "label"])
+
+        self.trainset = self.trainset.map(operations=transform, 
+                                          input_columns="image").batch(
+                                              batch_size=int(Context.get_parameters("batch_size", 32)))
+
+        self.testset = self.testset.map(operations=transform, 
+                                          input_columns="image").batch(
+                                              batch_size=int(Context.get_parameters("batch_size", 32)))
+
+class SddModel(nn.Cell):
+    def __init__(self):
+        super().__init__()
+        self.conv1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=(3, 3), stride=2)
+        self.relu1 = nn.ReLU()
+        self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)
+
+        self.conv2 = nn.Conv2d(in_channels=64, out_channels=32, kernel_size=(3, 3))
+        self.relu2 = nn.ReLU()
+        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)
+
+        self.flatten = nn.Flatten()
+
+        self.dropout = nn.Dropout(0.25)
+
+        self.fc1 = nn.Dense(8192, 64)
+        self.relu3 = nn.ReLU()
+
+        self.fc2 = nn.Dense(64, 32)
+        self.relu4 = nn.ReLU()
+
+        self.fc3 = nn.Dense(32, 2)
+
+    def construct(self, x):
+        x = self.pool1(self.relu1(self.conv1(x)))
+        x = self.pool2(self.relu2(self.conv2(x)))
+        x = self.dropout(self.flatten(x))
+        x = self.relu3(self.fc1(x))
+        x = self.relu4(self.fc2(x))
+        x = self.fc3(x)
+        return x
+
+class Estimator:
+    def __init__(self):
+        self.model = SddModel()
+        self.pretrained = None
+        self.saved = None
+        self.hyperparameters = {
+            "use_mindspore": True,
+            "type": "basic",
+            "rounds": int(Context.get_parameters("exit_round", 5)),
+            "target_accuracy": 0.97,
+            "epochs": int(Context.get_parameters("epochs", 5)),
+            "batch_size": int(Context.get_parameters("batch_size", 32)),
+            "optimizer": "SGD",
+            "learning_rate": float(Context.get_parameters("learning_rate", 0.01)),
+            # The machine learning model
+            "model_name": "sdd_model",
+            "momentum": 0.9,
+            "weight_decay": 0.0,
+            "history": 0.1
+        }
+
+

examples/federated_learning/surface_defect_detection_mindspore/train.py

@@ -0,0 +1,63 @@
+# Copyright 2021 The KubeEdge Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from PIL import Image
+
+from interface import fedavg, s3_transmitter
+from interface import myDataset, Estimator
+from sedna.core.federated_learning import FederatedLearningV2
+from sedna.datasources import TxtDataParse
+from sedna.common.config import BaseConfig
+
+def image_process(line):
+    file_path, label = line.split(',')
+    root_path = os.getcwd()
+    file_path = os.path.join(root_path, file_path)
+    x = Image.open(file_path)
+    y = int(label)
+    return [x, y]
+
+def readFromTxt(path):
+    data_x = []
+    data_y = []
+    with open(path) as f:
+        lines = f.readlines()
+        for line in lines:
+            x, y = image_process(line)
+            data_x.append(x)
+            data_y.append(y)
+    return data_x, data_y
+
+def main():
+    train_dataset_url = BaseConfig.train_dataset_url
+    # we have same data in the trainset and testset
+    test_dataset_url = BaseConfig.train_dataset_url
+
+    # train_data = TxtDataParse(data_type="train", func=image_process)
+    # train_data.parse(train_dataset_url)
+    train_data = readFromTxt(train_dataset_url)
+    data = myDataset(trainset=train_data, testset=train_data)
+
+    estimator = Estimator()
+
+    fl_model = FederatedLearningV2(
+        data=data,
+        estimator=estimator,
+        aggregation=fedavg,
+        transmitter=s3_transmitter)
+
+    fl_model.train()
+
+if __name__ == '__main__':
+    main()

examples/federated_learning/surface_defect_detection_pytorch/README.md

@@ -0,0 +1,177 @@
+# Using Federated Learning Job in Surface Defect Detection Scenario
+This case introduces how to use federated learning job in surface defect detection scenario.
+In the safety surface defect detection, data is scattered in different places (such as server node, camera or others) and cannot be aggregated due to data privacy and bandwidth. As a result, we cannot use all the data for training.
+Using Federated Learning, we can solve the problem. Each place uses its own data for model training ,uploads the weight to the cloud for aggregation, and obtains the aggregation result for model update.
+
+
+## Surface Defect Detection Experiment
+> Assume that there are two edge nodes and a cloud node. Data on the edge nodes cannot be migrated to the cloud due to privacy issues.
+> Base on this scenario, we will demonstrate the surface inspection.
+
+### Prepare Nodes
+```
+CLOUD_NODE="cloud-node-name"
+EDGE1_NODE="edge1-node-name"
+EDGE2_NODE="edge2-node-name"
+```
+
+### Install Sedna
+
+Follow the [Sedna installation document](/docs/setup/install.md) to install Sedna.
+
+### Prepare Dataset
+
+Download [dataset](https://github.com/abin24/Magnetic-tile-defect-datasets.) and the [label file](/examples/federated_learning/surface_defect_detection/data/1.txt) to `/data` of ```EDGE1_NODE```.  
+```
+mkdir -p /data
+cd /data
+git clone https://github.com/abin24/Magnetic-tile-defect-datasets..git Magnetic-tile-defect-datasets
+curl -o 1.txt https://raw.githubusercontent.com/kubeedge/sedna/main/examples/federated_learning/surface_defect_detection/data/1.txt
+```
+
+Download [dataset](https://github.com/abin24/Magnetic-tile-defect-datasets.) and the [label file](/examples/federated_learning/surface_defect_detection/data/2.txt) to `/data` of ```EDGE2_NODE```.
+```
+mkdir -p /data
+cd /data
+git clone https://github.com/abin24/Magnetic-tile-defect-datasets..git Magnetic-tile-defect-datasets
+curl -o 2.txt https://raw.githubusercontent.com/kubeedge/sedna/main/examples/federated_learning/surface_defect_detection/data/2.txt
+```
+
+### Prepare Images
+This example uses these images:
+1. aggregation worker: ```kubeedge/sedna-example-federated-learning-surface-defect-detection-aggregation:v0.3.0```
+2. train worker: ```kubeedge/sedna-example-federated-learning-surface-defect-detection-train:v0.3.0```
+
+These images are generated by the script [build_images.sh](/examples/build_image.sh).
+
+### Create Federated Learning Job 
+
+#### Create Dataset
+
+create dataset for `$EDGE1_NODE`
+```
+kubectl create -f - <<EOF
+apiVersion: sedna.io/v1alpha1
+kind: Dataset
+metadata:
+  name: "edge1-surface-defect-detection-dataset"
+spec:
+  url: "/data/1.txt"
+  format: "txt"
+  nodeName: $EDGE1_NODE
+EOF
+```
+
+create dataset for `$EDGE2_NODE`
+```
+kubectl create -f - <<EOF
+apiVersion: sedna.io/v1alpha1
+kind: Dataset
+metadata:
+  name: "edge2-surface-defect-detection-dataset"
+spec:
+  url: "/data/2.txt"
+  format: "txt"
+  nodeName: $EDGE2_NODE
+EOF
+```
+
+#### Create Model
+
+create the directory `/model` in the host of `$EDGE1_NODE`
+```
+mkdir /model
+```
+create the directory `/model` in the host of `$EDGE2_NODE`
+```
+mkdir /model
+```
+
+create model
+```
+kubectl create -f - <<EOF
+apiVersion: sedna.io/v1alpha1
+kind: Model
+metadata:
+  name: "surface-defect-detection-model"
+spec:
+  url: "/model"
+  format: "pb"
+EOF
+```
+
+#### Start Federated Learning Job
+
+```
+kubectl create -f - <<EOF
+apiVersion: sedna.io/v1alpha1
+kind: FederatedLearningJob
+metadata:
+  name: surface-defect-detection
+spec:
+  aggregationWorker:
+    model:
+      name: "surface-defect-detection-model"
+    template:
+      spec:
+        nodeName: $CLOUD_NODE
+        containers:
+          - image: kubeedge/sedna-example-federated-learning-surface-defect-detection-aggregation:v0.3.0
+            name:  agg-worker
+            imagePullPolicy: IfNotPresent
+            env: # user defined environments
+              - name: "exit_round"
+                value: "3"
+            resources:  # user defined resources
+              limits:
+                memory: 2Gi
+  trainingWorkers:
+    - dataset:
+        name: "edge1-surface-defect-detection-dataset"
+      template:
+        spec:
+          nodeName: $EDGE1_NODE
+          containers:
+            - image: kubeedge/sedna-example-federated-learning-surface-defect-detection-train:v0.3.0
+              name:  train-worker
+              imagePullPolicy: IfNotPresent
+              env:  # user defined environments
+                - name: "batch_size"
+                  value: "32"
+                - name: "learning_rate"
+                  value: "0.001"
+                - name: "epochs"
+                  value: "2"
+              resources:  # user defined resources
+                limits:
+                  memory: 2Gi
+    - dataset:
+        name: "edge2-surface-defect-detection-dataset"
+      template:
+        spec:
+          nodeName: $EDGE2_NODE
+          containers:
+            - image: kubeedge/sedna-example-federated-learning-surface-defect-detection-train:v0.3.0
+              name:  train-worker
+              imagePullPolicy: IfNotPresent
+              env:  # user defined environments
+                - name: "batch_size"
+                  value: "32"
+                - name: "learning_rate"
+                  value: "0.001"
+                - name: "epochs"
+                  value: "2"
+              resources:  # user defined resources
+                limits:
+                  memory: 2Gi
+EOF
+```
+
+### Check Federated Learning Status
+
+```
+kubectl get federatedlearningjob surface-defect-detection
+```
+
+### Check Federated Learning Train Result
+After the job completed, you will find the model generated on the directory `/model` in `$EDGE1_NODE` and `$EDGE2_NODE`.