Fast Cluster Restore

arangorestore
  --server.endpoint <endpoint-of-a-coordinator>
  --server.database <database-name>
  --server.password <password>
  --import-data false
  --input-directory <dump-directory>

echo 'db._collections().filter(function(c) { return c.name()[0] !== "_"; })
.forEach(function(c) { print("collection:", c.name(), "replicationFactor:",
c.properties().replicationFactor); c.properties({ replicationFactor: 1 }); });'
| arangosh
  --server.endpoint <endpoint-of-a-coordinator>
  --server.database <database-name>
  --server.username <user-name>
  --server.password <password>

#!/bin/sh
#
# Version: 0.3
#
# Release Notes:
# - v0.3: fixed a bug that was happening when the collection name included an underscore 
# - v0.2: compatibility with version 3.4: now each coordinator_<number-of-coordinator>.sh
#         includes a single restore command (instead of one for each collection)
#         which allows making using of the --threads option in v.3.4.0 and later
# - v0.1: initial version

if test -z "$ARANGOSH" ; then
  export ARANGOSH=arangosh
fi
cat > /tmp/parallelRestore$$.js <<'EOF'
var fs = require("fs");
var print = require("internal").print;
var exit = require("internal").exit;
var arangorestore = "arangorestore";
var env = require("internal").env;
if (env.hasOwnProperty("ARANGORESTORE")) {
  arangorestore = env["ARANGORESTORE"];
}

// Check ARGUMENTS: dumpDir coordinator1 coordinator2 ...

if (ARGUMENTS.length < 2) {
  print("Need at least two arguments DUMPDIR and COORDINATOR_ENDPOINTS!");
  exit(1);
}

var dumpDir = ARGUMENTS[0];
var coordinators = ARGUMENTS[1].split(",");
var otherArgs = ARGUMENTS.slice(2);

// Quickly check the dump dir:
var files = fs.list(dumpDir).filter(f => !fs.isDirectory(f));
var found = files.indexOf("ENCRYPTION");
if (found === -1) {
  print("This directory does not have an ENCRYPTION entry.");
  exit(2);
}
// Remove ENCRYPTION entry:
files = files.slice(0, found).concat(files.slice(found+1));

for (let i = 0; i < files.length; ++i) {
  if (files[i].slice(-5) !== ".json") {
    print("This directory has files which do not end in '.json'!");
    exit(3);
  }
}
files = files.map(function(f) {
  var fullName = fs.join(dumpDir, f);
  var collName = "";
  if (f.slice(-10) === ".data.json") {
    var pos;
    if (f.slice(0, 1) === "_") {  // system collection
      pos = f.slice(1).indexOf("_") + 1;
      collName = "_" + f.slice(1, pos);
    } else {
      pos = f.lastIndexOf("_")
      collName = f.slice(0, pos);
    }
  }
  return {name: fullName, collName, size: fs.size(fullName)};
});
files = files.sort(function(a, b) { return b.size - a.size; });
var dataFiles = [];
for (let i = 0; i < files.length; ++i) {
  if (files[i].name.slice(-10) === ".data.json") {
    dataFiles.push(i);
  }
}

// Produce the scripts, one for each coordinator:
var scripts = [];
var collections = [];
for (let i = 0; i < coordinators.length; ++i) {
  scripts.push([]);
  collections.push([]);
}

var cnum = 0;
var temp = '';
var collections = [];
for (let i = 0; i < dataFiles.length; ++i) {
  var f = files[dataFiles[i]];
  if (typeof collections[cnum] == 'undefined') {
    collections[cnum] = (`--collection ${f.collName}`);
  } else {
    collections[cnum] += (` --collection ${f.collName}`);
  }
  cnum += 1;
  if (cnum >= coordinators.length) {
    cnum = 0;
  }
}

var cnum = 0;
for (let i = 0; i < coordinators.length; ++i) {
  scripts[i].push(`${arangorestore} --input-directory ${dumpDir} --server.endpoint ${coordinators[i]} ` + collections[i] + ' ' + otherArgs.join(" "));
}

for (let i = 0; i < coordinators.length; ++i) {
  let f = "coordinator_" + i + ".sh";
  print("Writing file", f, "...");
  fs.writeFileSync(f, scripts[i].join("\n"));
}
EOF

${ARANGOSH} --javascript.execute /tmp/parallelRestore$$.js -- "$@"
rm /tmp/parallelRestore$$.js

./parallelRestore <dump-directory>
  tcp://<ip-of-coordinator1>:<port of coordinator1>,
  tcp://<ip-of-coordinator2>:<port of coordinator2>,
  tcp://<ip-of-coordinator3>:<port of coordinator3>
  --server.username <username>
  --server.password <password>
  --server.database <database_name>
  --create-collection false

echo 'db._collections().filter(function(c) { return c.name()[0] !== "_"; })
.forEach(function(c) { print("collection:", c.name(), "replicationFactor:",
c.properties().replicationFactor); c.properties({ replicationFactor: 2 }); });'
| arangosh
  --server.endpoint <endpoint-of-a-coordinator>
  --server.database <database-name>
  --server.username <user-name>
  --server.password <password>