diff --git a/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions.sql b/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions.sql index f54ba731..16e9e444 100644 --- a/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions.sql +++ b/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions.sql @@ -83,7 +83,7 @@ raw_vessel_positions_load as ( -- Données remontées par le microbatch sur stg_ from {{ ref('stg_vessel_positions') }} where TRUE and vessel_id not like 'UNKNOWN_MMSI=%' - {{ MMSI_filter }} + {{ MMSI_filter }} ), @@ -137,8 +137,7 @@ raw_vessel_positions as ( -- Données remontées par le microbatch sur stg_vesse position_point, cast(NULL as boolean) as rn from raw_vessel_positions_load - - {{ MMSI_filter }} + where TRUE {{ MMSI_filter }} union all -- on ajoute la « dernière » ligne de chaque navire select * from previous_positions diff --git a/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions_x_zones.sql b/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions_x_zones.sql index a56f7351..d78dee67 100644 --- a/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions_x_zones.sql +++ b/backend/dbt_trawlwatch/models/itm/positions/itm_vessel_positions_x_zones.sql @@ -64,9 +64,9 @@ positions_with_zones_candidates as ( -- Jointure des positions des navires avec from positions as pos inner join zec_list as zec on pos.vessel_id = zec.vessel_id - and pos.position_timestamp between zec.excursion_start_position_timestamp and zec.excursion_end_position_timestamp + and utils.safe_between(pos.position_timestamp, zec.excursion_start_position_timestamp, zec.excursion_end_position_timestamp) and pos.position_id = any(zec.excursion_position_ids) -) , +), positions_x_zones as ( -- Positions des navires dans les zones maritimes (croisement spatial) select distinct diff --git a/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql b/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql index e668c994..51ce8d53 100644 --- a/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql +++ b/backend/dbt_trawlwatch/models/itm/segments/itm_vessel_segments.sql @@ -118,8 +118,7 @@ vessel_positions as ( -- On conserve uniquement les positions correspondant à des excursions inner join ( select * from {{ ref('itm_vessel_excursions') }} ) as exc on pos.vessel_id = exc.vessel_id - and pos.position_timestamp between exc.excursion_start_position_timestamp and exc.excursion_end_position_timestamp - + and utils.safe_between(pos.position_timestamp, exc.excursion_start_position_timestamp, exc.excursion_end_position_timestamp) {{ MMSI_filter }} ), @@ -338,8 +337,8 @@ segment_metrics as ( round(coalesce(extract(epoch from position_timestamp - position_timestamp_prev), 0)::numeric,0) as segment_duration_s, round(coalesce(extract(epoch from position_timestamp - position_timestamp_prev)/3600, 0)::numeric,4) as segment_duration_h, - round(st_distance(position_point, position_point_prev)::numeric,1) as segment_distance_m, - round((st_distance(position_point, position_point_prev)/1852)::numeric,1) as segment_distance_nm, + round(st_distance(position_point, position_point_prev)::numeric,1) as segment_distance_m, + round((st_distance(position_point, position_point_prev)/1852)::numeric,1) as segment_distance_nm, position_course as segment_course_at_end, position_course_prev as segment_course_at_start, diff --git a/backend/dbt_trawlwatch/models/staging/stg_vessel_positions.sql b/backend/dbt_trawlwatch/models/staging/stg_vessel_positions.sql index 9e275c54..2c86cd36 100644 --- a/backend/dbt_trawlwatch/models/staging/stg_vessel_positions.sql +++ b/backend/dbt_trawlwatch/models/staging/stg_vessel_positions.sql @@ -119,9 +119,9 @@ join_spire_ais_and_vessels as ( -- On ne conserve que la dernière remontée AIS and ais.position_latitude is not NULL and ais.position_longitude is not NULL and ais.position_speed is not NULL - and ais.position_heading is not NULL - and ais.position_course is not NULL - and ais.position_rot is not NULL + -- and ais.position_heading is not NULL + -- and ais.position_course is not NULL + -- and ais.position_rot is not NULL and ais.position_timestamp is not NULL ) diff --git a/backend/dbt_trawlwatch/seeds/historical_dim_vessels.csv b/backend/dbt_trawlwatch/seeds/historical_dim_vessels.csv index 9e382bbd..8f63f630 100644 --- a/backend/dbt_trawlwatch/seeds/historical_dim_vessels.csv +++ b/backend/dbt_trawlwatch/seeds/historical_dim_vessels.csv @@ -356,7 +356,7 @@ FRA000899950;GLENAN;84.1;FRA;PS;;9322669;FMHD;FRA000899950;4000;0;2500;0;;;FMHD; ITA000025490;ANTONINO SIRRATO;29.26;ITA;OTB;;8649292;IZHM;ITA000025490;515.2;0;184;115;;;00MV00343M;20/03/2012;;ACTIVE;EU REGISTER ITA000025490;ANTONINO SIRRATO;29.26;ITA;OTB;;8649292;IZHM;ITA000025490;305.44;0;184;115;;;00MV00343M;01/02/2002;19/03/2012;ACTIVE;EU REGISTER ITA000025490;ANTONINO SIRRATO;29.26;ITA;OTB;;8649292;;ITA000025490;305.44;0;184;115;;;00MV00343M;28/12/2001;31/01/2002;ACTIVE;EU REGISTER -NLD200002594;KENNEDY;27.1;IRL;SX;;9250103;EIVS9;NLD200002594;367;0;200;0;;;WD135;2019-09-03;;ACTIVE;EU REGISTER -NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;SPR;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;2003-01-01;2019-09-02;ACTIVE;EU REGISTER -NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;2001-02-01;2002-12-31;ACTIVE;EU REGISTER -NLD200002594;CORNELIS TRIJNTJE;23.99;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;215;0;;;UK-135;2000-06-30;2001-01-31;ACTIVE;EU REGISTER \ No newline at end of file +NLD200002594;KENNEDY;27.1;IRL;SX;;9250103;EIVS9;NLD200002594;367;0;200;0;;;WD135;03/09/2019;;ACTIVE;EU REGISTER +NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;SPR;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;01/01/2003;02/09/2019;ACTIVE;EU REGISTER +NLD200002594;CORNELIS TRIJNTJE;27.1;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;200;0;;;UK-135;01/02/2001;31/12/2002;ACTIVE;EU REGISTER +NLD200002594;CORNELIS TRIJNTJE;23.99;NLD;PTB;;9250103;PFAY;NLD200002594;367;66;215;0;;;UK-135;30/06/2000;31/01/2001;ACTIVE;EU REGISTER \ No newline at end of file diff --git a/backend/dbt_trawlwatch/seeds/static_vessels_table.csv b/backend/dbt_trawlwatch/seeds/static_vessels_table.csv index 0724ab50..7190e703 100644 --- a/backend/dbt_trawlwatch/seeds/static_vessels_table.csv +++ b/backend/dbt_trawlwatch/seeds/static_vessels_table.csv @@ -1701,4 +1701,5 @@ PS046;617095000;EGALABUR;91;> 80 m;CPV;Purse seiner;9710995;NA;D4GX;NA;FALSE;exp PS046;312078000;EGALABUR;91;> 80 m;BLZ;Purse seiner;9710995;NA;V3OI5;NA;TRUE;active;gfw, marine traffic and spire;NA; ESP000021905;312590000;TXORI BERRI;81;> 80 m;BLZ;Purse seiner;9006033;NA;V3UO9;NA;FALSE;exported;gfw, marine traffic and spire;NA; ESP000021905;461000089;TXORI BERRI;81;> 80 m;OMN;Purse seiner;9006033;NA;A4BB5;NA;TRUE;active;gfw, marine traffic and spire;NA; -DEU002280300;218004830;JAN MARIA;88.1;> 80 m;DEU;Bottom trawler;9917397;DEU002280300;DDPQ;BX792;TRUE;active;fleet register;NA; \ No newline at end of file +DEU002280300;218004830;JAN MARIA;88.1;> 80 m;DEU;Bottom trawler;9917397;DEU002280300;DDPQ;BX792;TRUE;active;fleet register;NA; +NLD199301863;232024007;FRANK BONEFAAS;119.65;> 80 m;GBR;Pelagic trawler;9074951;NLD199301863;MFYP7;H72;TRUE;active;fleet register;NA; \ No newline at end of file diff --git a/etl.sh b/etl.sh deleted file mode 100755 index 4a27fc26..00000000 --- a/etl.sh +++ /dev/null @@ -1,9 +0,0 @@ -#! /bin/sh - -source /home/bas/venv/bin/activate -cd $APP_FOLDER - -python bloom/tasks/load_spire_data_from_api.py && \ -flock bloom/tasks/clean_positions.py --command " - python bloom/tasks/clean_positions.py && - python bloom/tasks/create_update_excursions_segments.py" \ No newline at end of file diff --git a/etl_demo.sh b/etl_demo.sh index b8007d9e..f1cf2a44 100755 --- a/etl_demo.sh +++ b/etl_demo.sh @@ -5,7 +5,7 @@ source /Users/marthevienne/12_bloom/backend/.venv/bin/activate [ ! -f /Users/marthevienne/12_bloom/backend/.env ] || export $(grep -v '^#' /Users/marthevienne/12_bloom/backend/.env | xargs) cd ./backend/dbt_trawlwatch -sleep 60 +sleep 30 dbt run --select observ_spire_ais_data_retrievals dbt run --select itm_vessel_last_raw_position diff --git a/frontend/app/map/page.tsx b/frontend/app/map/page.tsx index 6969d5ea..99400ece 100644 --- a/frontend/app/map/page.tsx +++ b/frontend/app/map/page.tsx @@ -61,7 +61,8 @@ export default function MapPage() { fetcher, { revalidateOnFocus: false, - revalidateOnReconnect: false, + revalidateOnReconnect: true, + revalidateOnMount: true, keepPreviousData: true, } ) @@ -151,15 +152,28 @@ export default function MapPage() { for (const excursion of vesselExcursions) { const segments = await getVesselSegments( vesselID, - excursion.excursion_id + excursion.excursion_id, + startDate, + endDate ) excursion.segments = segments.data + console.log(excursion.arrival_at == undefined) const timeByMPAZone = await getVesselTimeByZone({ vesselId: vesselID, category: ZoneCategory.AMP, - startAt: startDate ? startDate > new Date(excursion.departure_at) ? new Date(startDate) : new Date(excursion.departure_at) : undefined, - endAt: endDate ? endDate < new Date(excursion.arrival_at!) ? new Date(endDate) : new Date(excursion.arrival_at!) : undefined, + startAt: startDate + ? startDate > new Date(excursion.departure_at) + ? new Date(startDate) + : new Date(excursion.departure_at) + : undefined, + endAt: endDate + ? excursion.arrival_at != undefined + ? endDate < new Date(excursion.arrival_at!) + ? new Date(endDate!) + : new Date(excursion.arrival_at!) + : new Date(endDate!) + : undefined, }) excursion.timeByMPAZone = timeByMPAZone console.log("timeByMPAZone", timeByMPAZone) diff --git a/frontend/services/backend-rest-client.ts b/frontend/services/backend-rest-client.ts index 32047bea..dab6e41c 100644 --- a/frontend/services/backend-rest-client.ts +++ b/frontend/services/backend-rest-client.ts @@ -111,8 +111,17 @@ export function getVesselExcursionsExtracts( return axios.get(url) } -export function getVesselSegments(vesselId: string, excursionId: string) { - const url = `${BASE_URL}/vessels/${vesselId}/excursions/${excursionId}/segments` +export function getVesselSegments(vesselId: string, excursionId: string, startDate?: Date, + endDate?: Date) { + let queryParams: string[] = [] + if (startDate) { + queryParams.push(`start_at=${startDate.toISOString()}`) + } + if (endDate) { + // queryParams.push(`end_at=${endDate.toISOString()}`) + queryParams.push(`end_at=${endDate.toISOString()}`) + } + const url = `${BASE_URL}/vessels/${vesselId}/excursions/${excursionId}/segments${queryParams.length > 0 ? `?${queryParams.join("&")}` : ""}` console.log(`GET ${url}`) return axios.get(url) }