Technical Note: [FortiSOAR / Cybersponse Tricks'n'Tips] Migrating records between CyOPs instances running different software versions
Description
The following python script can be used to migrate records from one CyOPs instance to another, regardless of whether the two instances are on the same CyOPs version.
Prerequisites
- Each module which is to be migrated must have the same fields, field names, and picklists defined in the destination CyOPs instance as are defined in the source CyOPs instance. To ensure this is the case, prior to running this script, those modules' configurations should be exported from the source CyOPs instance and imported into the destination CyOPs instance. This is done from the respective CyOPs instances' Configuration Managers.
- The constants at the start of the script (lines 6-9) must be defined. These constants will be the username and password to use on both CyOPs instances, and the respective hostnames for each.
- The list of modules to be migrated (lines13-16) must be populated properly. The list can be modified to include whatever modules you wish to move between CyOPs instances, and as an example in the script it is pre-populated to migrate the Users, Alerts, and the Incidents modules. NOTE: When populating this list, make note of modules that have "Lookup" fields. If Module A has a Lookup field which looks up a record in Module B, then records from Module B MUST be migrated prior to Module A's migration. This can be done by placing Module B before Module A in the modules list. One common example of this case is the "Assigned To" field on the Alerts module, which is a lookup for the Users module. Because the Alerts which will be migrated will be Assigned to Users which may not exist on the destination CyOPs instance, the Users records must be migrated first
import requests import json import math # VALUES NEEDED: USERNAME = "" PASSWORD = "" SRC_HOSTNAME = "" DEST_HOSTNAME = "" LIMIT = 100 CHUNK_SIZE = 5 modules = [ 'people', 'alerts', 'incidents' ] class Migrator: def __init__(self, username, password): self.source_hostname = SRC_HOSTNAME self.destination_hostname = DEST_HOSTNAME self.credentials = { 'credentials': { 'loginid': username, 'password': password } } self.src_token = None self.dst_token = None self._authorize() self.records=[] self.skipped=[] self.limit = LIMIT self.chunksize = CHUNK_SIZE def _authorize(self): src_auth_response = requests.post('https://{}/auth/authenticate'.format(self.source_hostname), data=json.dumps(self.credentials), headers={}, verify=False) self.src_token = json.loads(src_auth_response.text)['token'] self.src_headers = {"Authorization": "Bearer {}".format(self.src_token)} dst_auth_response = requests.post('https://{}/auth/authenticate'.format(self.destination_hostname), data=json.dumps(self.credentials), headers={}, verify=False) print("Auth: old token is {}".format(self.dst_token)) self.dst_token = json.loads(dst_auth_response.text)['token'] self.dst_headers = {"Authorization": "Bearer {}".format(self.dst_token)} print("Auth: new token is {}".format(self.dst_token)) return True def migrate_all_records(self, module, page): print("Exporting {}, page {}".format(module, page)) record_request = requests.get('https://{}/api/3/{}?$limit={}&$page={}&$export=true'.format(self.source_hostname, module, self.limit, page), headers=self.src_headers, verify=False) if record_request.status_code == 401: self._authorize() record_request = requests.get('https://{}/api/3/{}?$limit={}&$page={}&$export=true'.format(self.source_hostname, module, self.limit, page), headers=self.src_headers, verify=False) if record_request.status_code == 404: print("{} not found. Skipping...".format(module)) self.skipped.append(module) return False record_request=record_request.json() pages = math.ceil(record_request['hydra:totalItems'] / self.limit) self.records.extend(record_request['hydra:member']) print("Exported {} total records...".format(len(self.records))) self.import_all_records() if pages > page: self.migrate_all_records(module, page+1) return True def import_all_records(self): pages = math.ceil(len(self.records) / self.chunksize) for page in range(1,pages+1): if page == 1: records = self.records[:page * self.chunksize] elif page == pages: records = self.records[(page-1)*self.chunksize:] else: records = self.records[(page-1)*self.chunksize:page*self.chunksize] import_request = requests.post('https://{}/api/3/'.format(self.destination_hostname), json=records, headers=self.dst_headers, verify=False) if import_request.status_code == 401: self._authorize() import_request = requests.post('https://{}/api/3/'.format(self.destination_hostname), json=records, headers=self.dst_headers, verify=False) print("Import Response: {}".format(import_request.status_code)) #print(import_request.text) print("Imported {} records...".format(page*self.chunksize)) self.records = [] return True def check_quantity(self, module): get_module = requests.get('https://{}/api/3/{}'.format(self.destination_hostname, module), headers=self.dst_headers, verify=False) if get_module.status_code == 401: self._authorize() get_module = requests.get('https://{}/api/3/{}'.format(self.destination_hostname, module), headers=self.dst_headers, verify=False) if get_module.status_code == 404: print("{} not found. Skipping...".format(module)) self.skipped.append(module) return False quantity = get_module.json()['hydra:totalItems'] return quantity mig = Migrator(USERNAME, PASSWORD) for module in modules: completed = mig.migrate_all_records(module, 1) for module in mig.skipped: print("Module {} was not found and was skipped".format(module)) 
