I have perfSONAR archive running on a central host that accepts measurements from many different test points. I have been upgrading from 4 to 5 with a fresh install of EL9 on the archive and test points. The latency and trace tests I have are working correctly, and I can see the results in Grafana on the archive host. Throughput tests however show "No data" and digging into it it seems they get caught by Logstash on the archiver with this error:
"status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"mapper [result.intervals.json.streams.start] cannot be changed from type [float] to [long]"}
I'm not too familiar with logstash, but it seems the first value for [result.intervals.json.streams.start] is 0, while the rest appear to be floats (ex. 1.000613). This seems to be the case for all the throughput results that come in from different test points, including ad-hoc tests I did for testing.
I have attached the full log below:
[2024-10-10T10:29:53,337][WARN ][logstash.outputs.opensearch][pscheduler][ebc1769dcc1b3ae6f540debcf942bdee347571bf88e46889de2d2c0a32db5fbb] Could not index event to OpenSearch. {:status=>400, :action=>["create", {:_id=>nil, :_index=>"pscheduler_throughput", :routing=>nil}, {"@version"=>"1", "result"=>{"succeeded"=>true, "throughput"=>9412822572.19139, "intervals"=>{"json"=>[{"streams"=>[{"throughput-bits"=>9403108585.348906, "stream-id"=>5, "tcp-window-size"=>2535448, "retransmits"=>26, "omitted"=>false, "throughput-bytes"=>1176109056, "end"=>1.000613, "start"=>0, "rtt"=>1841}], "summary"=>{"throughput-bits"=>9403108585.348906, "retransmits"=>26, "omitted"=>false, "end"=>1.000613, "start"=>0, "throughput-bytes"=>1176109056}}, {"streams"=>[{"throughput-bits"=>9415328811.1761, "stream-id"=>5, "tcp-window-size"=>2541240, "retransmits"=>4, "omitted"=>false, "throughput-bytes"=>1177419776, "end"=>2.001041, "start"=>1.000613, "rtt"=>2151}], "summary"=>{"throughput-bits"=>9415328811.1761, "retransmits"=>4, "omitted"=>false, "end"=>2.001041, "start"=>1.000613, "throughput-bytes"=>1177419776}}, {"streams"=>[{"throughput-bits"=>9410969600, "stream-id"=>5, "tcp-window-size"=>2807672, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1176371200, "end"=>3.001041, "start"=>2.001041, "rtt"=>2166}], "summary"=>{"throughput-bits"=>9410969600, "retransmits"=>0, "omitted"=>false, "end"=>3.001041, "start"=>2.001041, "throughput-bytes"=>1176371200}}, {"streams"=>[{"throughput-bits"=>9407814900.008556, "stream-id"=>5, "tcp-window-size"=>2814912, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1175977984, "end"=>4.001042, "start"=>3.001041, "rtt"=>2182}], "summary"=>{"throughput-bits"=>9407814900.008556, "retransmits"=>0, "omitted"=>false, "end"=>4.001042, "start"=>3.001041, "throughput-bytes"=>1175977984}}, {"streams"=>[{"throughput-bits"=>9416255632.919464, "stream-id"=>5, "tcp-window-size"=>2885864, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1176502272, "end"=>5.000592, "start"=>4.001042, "rtt"=>2434}], "summary"=>{"throughput-bits"=>9416255632.919464, "retransmits"=>0, "omitted"=>false, "end"=>5.000592, "start"=>4.001042, "throughput-bytes"=>1176502272}}, {"streams"=>[{"throughput-bits"=>9419659709.58787, "stream-id"=>5, "tcp-window-size"=>3063968, "retransmits"=>6, "omitted"=>false, "throughput-bytes"=>1177419776, "end"=>6.00056, "start"=>5.000592, "rtt"=>2414}], "summary"=>{"throughput-bits"=>9419659709.58787, "retransmits"=>6, "omitted"=>false, "end"=>6.00056, "start"=>5.000592, "throughput-bytes"=>1177419776}}, {"streams"=>[{"throughput-bits"=>9412733441.950294, "stream-id"=>5, "tcp-window-size"=>3066864, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1177157632, "end"=>7.001041, "start"=>6.00056, "rtt"=>2372}], "summary"=>{"throughput-bits"=>9412733441.950294, "retransmits"=>0, "omitted"=>false, "end"=>7.001041, "start"=>6.00056, "throughput-bytes"=>1177157632}}, {"streams"=>[{"throughput-bits"=>9407701630.08839, "stream-id"=>5, "tcp-window-size"=>3068312, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1175977984, "end"=>8.001054, "start"=>7.001041, "rtt"=>2358}], "summary"=>{"throughput-bits"=>9407701630.08839, "retransmits"=>0, "omitted"=>false, "end"=>8.001054, "start"=>7.001041, "throughput-bytes"=>1175977984}}, {"streams"=>[{"throughput-bits"=>9402759774.96186, "stream-id"=>5, "tcp-window-size"=>3075552, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1175322624, "end"=>9.001035, "start"=>8.001054, "rtt"=>2323}], "summary"=>{"throughput-bits"=>9402759774.96186, "retransmits"=>0, "omitted"=>false, "end"=>9.001035, "start"=>8.001054, "throughput-bytes"=>1175322624}}, {"streams"=>[{"throughput-bits"=>9415124063.04288, "stream-id"=>5, "tcp-window-size"=>3075552, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1177157632, "end"=>10.001262, "start"=>9.001035, "rtt"=>2275}], "summary"=>{"throughput-bits"=>9415124063.04288, "retransmits"=>0, "omitted"=>false, "end"=>10.001262, "start"=>9.001035, "throughput-bytes"=>1177157632}}]}, "retransmits"=>36}, "id"=>"7f7a1e46-ddb7-4eaa-8654-1940d63d99e6", "pscheduler"=>{"tool"=>"iperf3", "added"=>"2024-10-10T10:29:25-05:00", "run_href"=>"https://perf-sac-1s/pscheduler/tasks/d552ec29-24aa-4cf6-a484-270d97e92690/runs/7f7a1e46-ddb7-4eaa-8654-1940d63d99e6", "start_time"=>"2024-10-10T10:29:34-05:00", "task_id"=>"d552ec29-24aa-4cf6-a484-270d97e92690", "participants"=>["perf-sac-1s.host", "perf-byn-1s.host"], "test_checksum"=>"c1e78bddf2b032b11653a9057c47e5fd5d480312514c163d8be001bc1b828c23", "duration"=>19.0, "end_time"=>"2024-10-10T10:29:47-05:00", "task_href"=>"https://perf-sac-1s/pscheduler/tasks/d552ec29-24aa-4cf6-a484-270d97e92690", "run_id"=>"7f7a1e46-ddb7-4eaa-8654-1940d63d99e6"}, "tags"=>["_geoip_lookup_failure"], "meta"=>{"destination"=>{"hostname"=>"perf-byn-1s.host", "ip"=>"10.7.32.34", "geo"=>{"as"=>{}}}, "ip_version"=>4, "source"=>{"hostname"=>"perf-sac-1s.host", "ip"=>"10.7.32.2", "geo"=>{"as"=>{}}}}, "reference"=>nil, "test"=>{"spec"=>{"dest"=>"perf-byn-1s.host", "schema"=>1, "source"=>"perf-sac-1s.host"}, "type"=>"throughput"}, "@timestamp"=>2024-10-10T15:29:53.227741113Z, "schedule"=>{"slip"=>300.0}, "host"=>"127.0.0.1"}], :response=>{"create"=>{"_index"=>"pscheduler_throughput", "_id"=>"WtkMd5IB-Lsh-6uL_rq2", "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"mapper [result.intervals.json.streams.start] cannot be changed from type [float] to [long]"}}}}
I have perfSONAR archive running on a central host that accepts measurements from many different test points. I have been upgrading from 4 to 5 with a fresh install of EL9 on the archive and test points. The latency and trace tests I have are working correctly, and I can see the results in Grafana on the archive host. Throughput tests however show "No data" and digging into it it seems they get caught by Logstash on the archiver with this error:
"status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"mapper [result.intervals.json.streams.start] cannot be changed from type [float] to [long]"}
I'm not too familiar with logstash, but it seems the first value for [result.intervals.json.streams.start] is 0, while the rest appear to be floats (ex. 1.000613). This seems to be the case for all the throughput results that come in from different test points, including ad-hoc tests I did for testing.
I have attached the full log below:
[2024-10-10T10:29:53,337][WARN ][logstash.outputs.opensearch][pscheduler][ebc1769dcc1b3ae6f540debcf942bdee347571bf88e46889de2d2c0a32db5fbb] Could not index event to OpenSearch. {:status=>400, :action=>["create", {:_id=>nil, :_index=>"pscheduler_throughput", :routing=>nil}, {"@version"=>"1", "result"=>{"succeeded"=>true, "throughput"=>9412822572.19139, "intervals"=>{"json"=>[{"streams"=>[{"throughput-bits"=>9403108585.348906, "stream-id"=>5, "tcp-window-size"=>2535448, "retransmits"=>26, "omitted"=>false, "throughput-bytes"=>1176109056, "end"=>1.000613, "start"=>0, "rtt"=>1841}], "summary"=>{"throughput-bits"=>9403108585.348906, "retransmits"=>26, "omitted"=>false, "end"=>1.000613, "start"=>0, "throughput-bytes"=>1176109056}}, {"streams"=>[{"throughput-bits"=>9415328811.1761, "stream-id"=>5, "tcp-window-size"=>2541240, "retransmits"=>4, "omitted"=>false, "throughput-bytes"=>1177419776, "end"=>2.001041, "start"=>1.000613, "rtt"=>2151}], "summary"=>{"throughput-bits"=>9415328811.1761, "retransmits"=>4, "omitted"=>false, "end"=>2.001041, "start"=>1.000613, "throughput-bytes"=>1177419776}}, {"streams"=>[{"throughput-bits"=>9410969600, "stream-id"=>5, "tcp-window-size"=>2807672, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1176371200, "end"=>3.001041, "start"=>2.001041, "rtt"=>2166}], "summary"=>{"throughput-bits"=>9410969600, "retransmits"=>0, "omitted"=>false, "end"=>3.001041, "start"=>2.001041, "throughput-bytes"=>1176371200}}, {"streams"=>[{"throughput-bits"=>9407814900.008556, "stream-id"=>5, "tcp-window-size"=>2814912, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1175977984, "end"=>4.001042, "start"=>3.001041, "rtt"=>2182}], "summary"=>{"throughput-bits"=>9407814900.008556, "retransmits"=>0, "omitted"=>false, "end"=>4.001042, "start"=>3.001041, "throughput-bytes"=>1175977984}}, {"streams"=>[{"throughput-bits"=>9416255632.919464, "stream-id"=>5, "tcp-window-size"=>2885864, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1176502272, "end"=>5.000592, "start"=>4.001042, "rtt"=>2434}], "summary"=>{"throughput-bits"=>9416255632.919464, "retransmits"=>0, "omitted"=>false, "end"=>5.000592, "start"=>4.001042, "throughput-bytes"=>1176502272}}, {"streams"=>[{"throughput-bits"=>9419659709.58787, "stream-id"=>5, "tcp-window-size"=>3063968, "retransmits"=>6, "omitted"=>false, "throughput-bytes"=>1177419776, "end"=>6.00056, "start"=>5.000592, "rtt"=>2414}], "summary"=>{"throughput-bits"=>9419659709.58787, "retransmits"=>6, "omitted"=>false, "end"=>6.00056, "start"=>5.000592, "throughput-bytes"=>1177419776}}, {"streams"=>[{"throughput-bits"=>9412733441.950294, "stream-id"=>5, "tcp-window-size"=>3066864, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1177157632, "end"=>7.001041, "start"=>6.00056, "rtt"=>2372}], "summary"=>{"throughput-bits"=>9412733441.950294, "retransmits"=>0, "omitted"=>false, "end"=>7.001041, "start"=>6.00056, "throughput-bytes"=>1177157632}}, {"streams"=>[{"throughput-bits"=>9407701630.08839, "stream-id"=>5, "tcp-window-size"=>3068312, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1175977984, "end"=>8.001054, "start"=>7.001041, "rtt"=>2358}], "summary"=>{"throughput-bits"=>9407701630.08839, "retransmits"=>0, "omitted"=>false, "end"=>8.001054, "start"=>7.001041, "throughput-bytes"=>1175977984}}, {"streams"=>[{"throughput-bits"=>9402759774.96186, "stream-id"=>5, "tcp-window-size"=>3075552, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1175322624, "end"=>9.001035, "start"=>8.001054, "rtt"=>2323}], "summary"=>{"throughput-bits"=>9402759774.96186, "retransmits"=>0, "omitted"=>false, "end"=>9.001035, "start"=>8.001054, "throughput-bytes"=>1175322624}}, {"streams"=>[{"throughput-bits"=>9415124063.04288, "stream-id"=>5, "tcp-window-size"=>3075552, "retransmits"=>0, "omitted"=>false, "throughput-bytes"=>1177157632, "end"=>10.001262, "start"=>9.001035, "rtt"=>2275}], "summary"=>{"throughput-bits"=>9415124063.04288, "retransmits"=>0, "omitted"=>false, "end"=>10.001262, "start"=>9.001035, "throughput-bytes"=>1177157632}}]}, "retransmits"=>36}, "id"=>"7f7a1e46-ddb7-4eaa-8654-1940d63d99e6", "pscheduler"=>{"tool"=>"iperf3", "added"=>"2024-10-10T10:29:25-05:00", "run_href"=>"https://perf-sac-1s/pscheduler/tasks/d552ec29-24aa-4cf6-a484-270d97e92690/runs/7f7a1e46-ddb7-4eaa-8654-1940d63d99e6", "start_time"=>"2024-10-10T10:29:34-05:00", "task_id"=>"d552ec29-24aa-4cf6-a484-270d97e92690", "participants"=>["perf-sac-1s.host", "perf-byn-1s.host"], "test_checksum"=>"c1e78bddf2b032b11653a9057c47e5fd5d480312514c163d8be001bc1b828c23", "duration"=>19.0, "end_time"=>"2024-10-10T10:29:47-05:00", "task_href"=>"https://perf-sac-1s/pscheduler/tasks/d552ec29-24aa-4cf6-a484-270d97e92690", "run_id"=>"7f7a1e46-ddb7-4eaa-8654-1940d63d99e6"}, "tags"=>["_geoip_lookup_failure"], "meta"=>{"destination"=>{"hostname"=>"perf-byn-1s.host", "ip"=>"10.7.32.34", "geo"=>{"as"=>{}}}, "ip_version"=>4, "source"=>{"hostname"=>"perf-sac-1s.host", "ip"=>"10.7.32.2", "geo"=>{"as"=>{}}}}, "reference"=>nil, "test"=>{"spec"=>{"dest"=>"perf-byn-1s.host", "schema"=>1, "source"=>"perf-sac-1s.host"}, "type"=>"throughput"}, "@timestamp"=>2024-10-10T15:29:53.227741113Z, "schedule"=>{"slip"=>300.0}, "host"=>"127.0.0.1"}], :response=>{"create"=>{"_index"=>"pscheduler_throughput", "_id"=>"WtkMd5IB-Lsh-6uL_rq2", "status"=>400, "error"=>{"type"=>"illegal_argument_exception", "reason"=>"mapper [result.intervals.json.streams.start] cannot be changed from type [float] to [long]"}}}}