Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate chain mask list config to full shard id list #831

Merged
merged 4 commits into from
Mar 24, 2020
Merged

Migrate chain mask list config to full shard id list #831

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cc8851c
Migrate chain mask list config to full shard id list
ninjaahhh Mar 12, 2020
b69de6e
Update config and fix redundant code
ninjaahhh Mar 12, 2020
b7272d9
Backward compatable
ninjaahhh Mar 23, 2020
a47286a
Address comments
ninjaahhh Mar 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions devnet/singularity/cluster_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,35 +380,35 @@
"HOST": "127.0.0.1",
"PORT": 38000,
"ID": "S0",
"CHAIN_MASK_LIST": [
4
"FULL_SHARD_ID_LIST": [
"0x1", "0x00040001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38001,
"WEBSOCKET_JSON_RPC_PORT": 38591,
"ID": "S1",
"CHAIN_MASK_LIST": [
5
"FULL_SHARD_ID_LIST": [
"0x00010001", "0x00050001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38002,
"WEBSOCKET_JSON_RPC_PORT": 38592,
"ID": "S2",
"CHAIN_MASK_LIST": [
6
"FULL_SHARD_ID_LIST": [
"0x00020001", "0x00060001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38003,
"WEBSOCKET_JSON_RPC_PORT": 38593,
"ID": "S3",
"CHAIN_MASK_LIST": [
7
"FULL_SHARD_ID_LIST": [
"0x00030001", "0x00070001"
]
}
],
Expand Down
16 changes: 8 additions & 8 deletions mainnet/singularity/cluster_config_bootnodes.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -456,32 +456,32 @@
"HOST": "127.0.0.1",
"PORT": 38000,
"ID": "S0",
"CHAIN_MASK_LIST": [
4
"FULL_SHARD_ID_LIST": [
"0x1", "0x00040001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38001,
"ID": "S1",
"CHAIN_MASK_LIST": [
5
"FULL_SHARD_ID_LIST": [
"0x00010001", "0x00050001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38002,
"ID": "S2",
"CHAIN_MASK_LIST": [
6
"FULL_SHARD_ID_LIST": [
"0x00020001", "0x00060001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38003,
"ID": "S3",
"CHAIN_MASK_LIST": [
7
"FULL_SHARD_ID_LIST": [
"0x00030001", "0x00070001"
]
}
],
Expand Down
16 changes: 8 additions & 8 deletions mainnet/singularity/cluster_config_p2p_crawling.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -456,32 +456,32 @@
"HOST": "127.0.0.1",
"PORT": 38000,
"ID": "S0",
"CHAIN_MASK_LIST": [
4
"FULL_SHARD_ID_LIST": [
"0x1", "0x00040001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38001,
"ID": "S1",
"CHAIN_MASK_LIST": [
5
"FULL_SHARD_ID_LIST": [
"0x00010001", "0x00050001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38002,
"ID": "S2",
"CHAIN_MASK_LIST": [
6
"FULL_SHARD_ID_LIST": [
"0x00020001", "0x00060001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38003,
"ID": "S3",
"CHAIN_MASK_LIST": [
7
"FULL_SHARD_ID_LIST": [
"0x00030001", "0x00070001"
]
}
],
Expand Down
16 changes: 8 additions & 8 deletions mainnet/singularity/cluster_config_template.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,32 +461,32 @@
"HOST": "127.0.0.1",
"PORT": 38000,
"ID": "S0",
"CHAIN_MASK_LIST": [
4
"FULL_SHARD_ID_LIST": [
"0x1", "0x00040001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38001,
"ID": "S1",
"CHAIN_MASK_LIST": [
5
"FULL_SHARD_ID_LIST": [
"0x00010001", "0x00050001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38002,
"ID": "S2",
"CHAIN_MASK_LIST": [
6
"FULL_SHARD_ID_LIST": [
"0x00020001", "0x00060001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38003,
"ID": "S3",
"CHAIN_MASK_LIST": [
7
"FULL_SHARD_ID_LIST": [
"0x00030001", "0x00070001"
]
}
],
Expand Down
32 changes: 16 additions & 16 deletions mainnet/singularity/cluster_config_template_8nodes.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -461,64 +461,64 @@
"HOST": "127.0.0.1",
"PORT": 38000,
"ID": "S0",
"CHAIN_MASK_LIST": [
8
"FULL_SHARD_ID_LIST": [
"0x1"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38001,
"ID": "S1",
"CHAIN_MASK_LIST": [
9
"FULL_SHARD_ID_LIST": [
"0x00010001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38002,
"ID": "S2",
"CHAIN_MASK_LIST": [
10
"FULL_SHARD_ID_LIST": [
"0x00020001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38003,
"ID": "S3",
"CHAIN_MASK_LIST": [
11
"FULL_SHARD_ID_LIST": [
"0x00030001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38004,
"ID": "S4",
"CHAIN_MASK_LIST": [
12
"FULL_SHARD_ID_LIST": [
"0x00040001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38005,
"ID": "S5",
"CHAIN_MASK_LIST": [
13
"FULL_SHARD_ID_LIST": [
"0x00050001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38006,
"ID": "S6",
"CHAIN_MASK_LIST": [
14
"FULL_SHARD_ID_LIST": [
"0x00060001"
]
},
{
"HOST": "127.0.0.1",
"PORT": 38007,
"ID": "S7",
"CHAIN_MASK_LIST": [
15
"FULL_SHARD_ID_LIST": [
"0x00070001"
]
}
],
Expand Down
31 changes: 22 additions & 9 deletions quarkchain/cluster/cluster_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from quarkchain.cluster.monitoring import KafkaSampleLogger
from quarkchain.cluster.rpc import SlaveInfo
from quarkchain.config import BaseConfig, ChainConfig, QuarkChainConfig
from quarkchain.core import Address, ChainMask
from quarkchain.core import Address
from quarkchain.utils import Logger, check, is_p2

DEFAULT_HOST = socket.gethostbyname(socket.gethostname())
Expand Down Expand Up @@ -91,17 +91,21 @@ class SlaveConfig(BaseConfig):
PORT = 38392
WEBSOCKET_JSON_RPC_PORT = None
ID = ""
CHAIN_MASK_LIST = None
FULL_SHARD_ID_LIST = []

def to_dict(self):
ret = super().to_dict()
ret["CHAIN_MASK_LIST"] = [m.value for m in self.CHAIN_MASK_LIST]
# format to hex
ret["FULL_SHARD_ID_LIST"] = [
"0x{:08x}".format(i) for i in self.FULL_SHARD_ID_LIST
]
return ret

@classmethod
def from_dict(cls, d):
config = super().from_dict(d)
config.CHAIN_MASK_LIST = [ChainMask(v) for v in config.CHAIN_MASK_LIST]
ninjaahhh marked this conversation as resolved.
Show resolved Hide resolved
# parse from hex to int
config.FULL_SHARD_ID_LIST = [int(h, 16) for h in config.FULL_SHARD_ID_LIST]
return config


Expand All @@ -112,7 +116,7 @@ class SimpleNetworkConfig(BaseConfig):

class P2PConfig(BaseConfig):
# *new p2p module*
BOOT_NODES = "" # comma seperated enodes format: enode://PUBKEY@IP:PORT
BOOT_NODES = "" # comma separated enodes format: enode://PUBKEY@IP:PORT
PRIV_KEY = ""
MAX_PEERS = 25
UPNP = False
Expand Down Expand Up @@ -171,7 +175,7 @@ def __init__(self):
slave_config = SlaveConfig()
slave_config.PORT = 38000
slave_config.ID = "S0"
slave_config.CHAIN_MASK_LIST = [ChainMask(1)]
slave_config.FULL_SHARD_ID_LIST = [2]
ninjaahhh marked this conversation as resolved.
Show resolved Hide resolved
self.SLAVE_LIST.append(slave_config)

fd, self.json_filepath = tempfile.mkstemp()
Expand All @@ -182,7 +186,7 @@ def get_slave_info_list(self):
results = []
for slave in self.SLAVE_LIST:
results.append(
SlaveInfo(slave.ID, slave.HOST, slave.PORT, slave.CHAIN_MASK_LIST)
SlaveInfo(slave.ID, slave.HOST, slave.PORT, slave.FULL_SHARD_ID_LIST)
)
return results

Expand Down Expand Up @@ -414,10 +418,19 @@ def __create_from_args_internal():
slave_config = SlaveConfig()
slave_config.PORT = args.port_start + i
slave_config.ID = "S{}".format(i)
slave_config.CHAIN_MASK_LIST = [ChainMask(i | args.num_slaves)]

slave_config.FULL_SHARD_ID_LIST = []
config.SLAVE_LIST.append(slave_config)

# assign full shard IDs to each slave, using hex strings to write into JSON
full_shard_ids = [
(i << 16) + args.num_shards_per_chain + j
for i in range(args.num_chains)
for j in range(args.num_shards_per_chain)
]
for i, full_shard_id in enumerate(full_shard_ids):
slave = config.SLAVE_LIST[i % args.num_slaves]
slave.FULL_SHARD_ID_LIST.append(full_shard_id)

fd, config.json_filepath = tempfile.mkstemp()
with os.fdopen(fd, "w") as tmp:
tmp.write(config.to_json())
Expand Down
Loading