Resume/Continue failing

[yakirgb]

Hi @adejanovski ,
Continue failing with Python error:

10:37:25 unbuffer /usr/local/bin/cstar continue 470ef4f5-d6a2-4372-a9e6-2430c30e22a9 -v 2>&1 | tee -a /var/log/cstar/cstar_RESUME_20210210-083724.log
10:37:25 Retry :  False
10:37:25 Resuming job 470ef4f5-d6a2-4372-a9e6-2430c30e22a9
10:37:25 Running  /usr/local/lib/python3.6/site-packages/cstar/resources/commands/run.sh
10:37:54 Traceback (most recent call last):
10:37:54   File "/usr/local/bin/cstar", line 8, in <module>
10:37:54     sys.exit(main())
10:37:54   File "/usr/local/lib/python3.6/site-packages/cstar/cstarcli.py", line 225, in main
10:37:54     namespace.func(namespace)
10:37:54   File "/usr/local/lib/python3.6/site-packages/cstar/cstarcli.py", line 70, in execute_continue
10:37:54     job.resume()
10:37:54   File "/usr/local/lib/python3.6/site-packages/cstar/job.py", line 332, in resume
10:37:54     self.run()
10:37:54   File "/usr/local/lib/python3.6/site-packages/cstar/job.py", line 344, in run
10:37:54     self.schedule_all_runnable_jobs()
10:37:54   File "/usr/local/lib/python3.6/site-packages/cstar/job.py", line 433, in schedule_all_runnable_jobs
10:37:54     next_host = self.state.find_next_host()
10:37:54   File "/usr/local/lib/python3.6/site-packages/cstar/state.py", line 75, in find_next_host
10:37:54     ignore_down_nodes=self.ignore_down_nodes)
10:37:54   File "/usr/local/lib/python3.6/site-packages/cstar/strategy.py", line 69, in find_next_host
10:37:54     return _strategy_mapping[strategy](remaining, endpoint_mapping, progress.running)
10:37:54   File "/usr/local/lib/python3.6/site-packages/cstar/strategy.py", line 83, in _topology_find_next_host
10:37:54     for next in endpoint_mapping[h]:
10:37:54 KeyError: Host(fqdn='1.1.1.1', ip='1.1.1.1', dc='UND', cluster='UND Cluster', rack='RAC1', is_up=True, host_id='259fc144-0a35-4e92-9cb6-09f099c911df')
12:32:46 
12:32:46 Aborted

job.json file looks like:

{
    "cache_directory": "/var/lib/jk/.cstar/cache",
    "command": "/usr/local/lib/python3.6/site-packages/cstar/resources/commands/run.sh",
    "creation_timestamp": 1612946274,
    "env": {
        "COMMAND": "/home/cassandra/scripts/cassandra_restart.sh \"/home/cassandra/scripts/find_expired_files.sh\""
    },
    "errors": [],
    "hosts_variables": {},
    "is_preheated": false,
    "jmx_username": null,
    "job_runner": "RemoteJobRunner",
    "key_space": null,
    "output_directory": "/var/lib/jk/.cstar/jobs/470ef4f5-d6a2-4372-a9e6-2430c30e22a9",
    "resolve_hostnames": false,
    "returned_jobs": [],
    "schema_versions": [
        "b1066b3a-c020-3e13-afde-b977369eb723"
    ],
    "sleep_after_done": null,
    "sleep_on_new_runner": 0.5,
    "ssh_identity_file": "/var/lib/jk/.ssh/id_rsa",
    "ssh_lib": "ssh2",
    "ssh_password": null,
    "ssh_username": "root",
    "state": {
        "cluster_parallel": true,
        "current_topology": [

....long list
                [
                    "4.4.4.4",
                    "4.4.4.4",
                    "UND",
                    "UND Cluster",
                    "RAC1",
                    true,
                    "9544d7e9-ebdc-40fa-9532-ca37b0ed7d80"
                ],
                [
                    "3.3.3.3",
                    "3.3.3.3",
                    "UND",
                    "UND Cluster",
                    "RAC1",
                    true,
                    "4e8b75c9-e3c0-4447-bc5f-a5198a51d416"
                ]
            ],
            "failed": [
                [
                    "2.2.2.2",
                    "2.2.2.2",
                    "UND",
                    "UND Cluster",
                    "RAC1",
                    true,
                    "52660ead-5bd4-4bb5-9d27-7dc5f31b8d1a"
                ]
            ],
            "running": [
                [
                    "1.1.1.1",
                    "1.1.1.1",
                    "UND",
                    "UND Cluster",
                    "RAC1",
                    true,
                    "259fc144-0a35-4e92-9cb6-09f099c911df"
                ]
            ]
        },
        "strategy": "topology"
    },
    "status_topology_hash": [
        "1bac2bdafcc8201f0fbf2f1f023f8cba"
    ],
    "timeout": null,
    "version": 8
}

Thank you, Yakir Gibraltar

[yakirgb]

Related to: #62 (comment)

[yakirgb]

@adejanovski the issue related to function get_cache_file_path:

    def get_cache_file_path(self, cache_type):
        debug("Cache file: {}-{}-{}".format(cache_type, "-".join(sorted(self.schema_versions)), "-".join(sorted(self.status_topology_hash))))
        return os.path.join(self.cache_directory, "{}-{}-{}".format(cache_type, "-".join(sorted(self.schema_versions)), "-".join(sorted(self.status_topology_hash))))

The values of self.status_topology_hash and self.schema_versions are empty
But the values of status_topology_hash and schema_versions in job.json file !
And it's causing read from incorrect and old cache file: Cache file: endpoint_mapping--

Why not use endpoint_mapping with prefix of endpoint_mapping-<job_id> ?

As a workaround i just removed file /var/lib/jenkins/.cstar/cache/endpoint_mapping--

TL;DR
cstar unable use cache file on continue because missing values of schema_versions and status_topology_hash

[yakirgb]

Another issue that related to continue :
Cstar continue failing with error of (replaced the real ip..):

/usr/local/bin/cstar continue 324ca2be-169f-4cx7-b7f7-334b68edf69c -vvv
.....
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/site-packages/cstar/cstarcli.py", line 58, in execute_continue
    output_directory=args.output_directory, retry=args.retry_failed)
  File "/usr/local/lib/python3.6/site-packages/cstar/jobreader.py", line 35, in read
    return _parse(f.read(), file, output_directory, job, job_id, stop_after, max_days, endpoint_mapper, retry)
  File "/usr/local/lib/python3.6/site-packages/cstar/jobreader.py", line 94, in _parse
    endpoint_mapping = endpoint_mapper(original_topology)
  File "/usr/local/lib/python3.6/site-packages/cstar/job.py", line 208, in get_endpoint_mapping
    mappings.append(cstar.endpoint_mapping.parse(range_mapping, topology, lookup=ip_lookup))
  File "/usr/local/lib/python3.6/site-packages/cstar/endpoint_mapping.py", line 23, in parse
    return _topology_map(_dns_map(_parse_range_mapping(range_mapping), lookup), topology)
  File "/usr/local/lib/python3.6/site-packages/cstar/endpoint_mapping.py", line 57, in _topology_map
    host = topology.get_host(raw_host)
  File "/usr/local/lib/python3.6/site-packages/cstar/topology.py", line 54, in get_host
    raise UnknownHost(ip)
cstar.exceptions.UnknownHost: Unknown host: 10.20.30.40
Error: Unknown host: 10.20.30.40

The host in job.json looks like:

[
    "10.20.30.40",
    "10.20.30.40,
    "abc",
    "D Cluster",
    "RAC1",
    true,
    "e3859160-36af-4df9-a104-933e193adff0"
],

The hosts 10.20.30.40 is part of the cluster and resolving working without problems
Any ideas? maybe related to default "resolve_hostnames": false ?