python /home/admin/mtr/script_for_cron.py -j default -m 20 -a 'python3 /home/admin/workarea/git/Velours/python/prod/memo/memo.py --job=sla_mensuel --raspi=20090 --dashboard_places=cedre --rootfolder_sla=/data_2/data_log/job/memo/cedre -v --mtr_id=1080 --date=`date -d"$nowdate 15 day ago" +%Y-%m` ' -s sla_mensuel -M 0 -S 0 -U 90,80,95 import MySQLdb succeeded Import error (python version) Need env var : LOGRASPI Raspi : 20090, Date : 2025-03 _cr_cnf_ : /home/admin/mtr/.credentials/credentials.py _cr_cnf_ : /home/admin/mtr/.credentials/credentials.py warning , we can't find thcl infos in json_data warning , we can't find pdt infos in json_data nb_day : (5, 31) VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4346 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4346 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4346 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4346 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) DONE and to test : checkNoCycle ! We are managing only one step so we do not consider checkConsistencyNbInputNbOutput ! We are managing only one step so we do not consider checkConsistencyTypeOutputInput ! {"input_values":[]} SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4510 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4510 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4510 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4510 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! WARNING : number of outputs for step 12695 mask_detect is not consistent : 4 used against 2 in the step definition ! WARNING : number of outputs for step 12698 rle_unique_nms_with_priority is not consistent : 2 used against 1 in the step definition ! Step 12702 crop_condition have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! WARNING : number of outputs for step 12702 crop_condition is not consistent : 4 used against 3 in the step definition ! WARNING : number of outputs for step 12699 ventilate_hashtags_in_portfolio is not consistent : 2 used against 1 in the step definition ! Step 12700 final have less inputs used (2) than in the step definition (3) : maybe we manage optionnal inputs ! Step 12700 final have less outputs used (1) than in the step definition (2) : some outputs may be not used ! Step 12704 split_time_score have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 2 of step 12701 have datatype=6 We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 12699 doesn't seem to be define in the database( WARNING : type of input 3 of step 12700 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 0 of step 12703 have datatype=18 WARNING : type of input 5 of step 12701 doesn't seem to be define in the database( WARNING : output 0 of step 12703 have datatype=11 whereas input 5 of step 12701 have datatype=None WARNING : output 0 of step 12701 have datatype=10 whereas input 0 of step 12704 have datatype=18 WARNING : type of input 2 of step 12702 doesn't seem to be define in the database( WARNING : output 1 of step 12698 have datatype=7 whereas input 2 of step 12702 have datatype=None WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12697 doesn't seem to be define in the database( WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12696 doesn't seem to be define in the database( WARNING : type of output 3 of step 12702 doesn't seem to be define in the database( WARNING : type of input 1 of step 12699 doesn't seem to be define in the database( DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'name_pipeline': 'aggregate_carac_ratio', 'hashtag_type': 4549, 'hashtag_proportion': 'sac,textiles,verre,organique,dasri,masque,encombrant,autre_emballage,autre_non_emballage,film_plastique,sac_noir,sac_transparent', 'hashtag_parmi': 'entrant', 'hashtag_weights': {'sac': 6.0, 'textiles': 3.0, 'verre': 3.0, 'organique': 10.0, 'dasri': 10.0, 'masque': 3.0, 'encombrant': 10.0, 'autre_emballage': 3.0, 'autre_non_emballage': 3.0, 'entrant': 1.0, 'sac_noir': 1.0, 'sac_transparent': 1.0, 'film_plastique': 1.0}} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier TODO : Insert select and so on SELECT id, name, misc_info, datou_id_for_right FROM MTRPhoto.dashboard_places WHERE name="cedre"; SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4346 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4346 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4346 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4346 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) DONE and to test : checkNoCycle ! We are managing only one step so we do not consider checkConsistencyNbInputNbOutput ! We are managing only one step so we do not consider checkConsistencyTypeOutputInput ! {"input_values":[]} SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4510 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4510 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4510 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4510 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! WARNING : number of outputs for step 12695 mask_detect is not consistent : 4 used against 2 in the step definition ! WARNING : number of outputs for step 12698 rle_unique_nms_with_priority is not consistent : 2 used against 1 in the step definition ! Step 12702 crop_condition have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! WARNING : number of outputs for step 12702 crop_condition is not consistent : 4 used against 3 in the step definition ! WARNING : number of outputs for step 12699 ventilate_hashtags_in_portfolio is not consistent : 2 used against 1 in the step definition ! Step 12700 final have less inputs used (2) than in the step definition (3) : maybe we manage optionnal inputs ! Step 12700 final have less outputs used (1) than in the step definition (2) : some outputs may be not used ! Step 12704 split_time_score have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 2 of step 12701 have datatype=6 We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 12699 doesn't seem to be define in the database( WARNING : type of input 3 of step 12700 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 0 of step 12703 have datatype=18 WARNING : type of input 5 of step 12701 doesn't seem to be define in the database( WARNING : output 0 of step 12703 have datatype=11 whereas input 5 of step 12701 have datatype=None WARNING : output 0 of step 12701 have datatype=10 whereas input 0 of step 12704 have datatype=18 WARNING : type of input 2 of step 12702 doesn't seem to be define in the database( WARNING : output 1 of step 12698 have datatype=7 whereas input 2 of step 12702 have datatype=None WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12697 doesn't seem to be define in the database( WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12696 doesn't seem to be define in the database( WARNING : type of output 3 of step 12702 doesn't seem to be define in the database( WARNING : type of input 1 of step 12699 doesn't seem to be define in the database( DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'name_pipeline': 'aggregate_carac_ratio', 'hashtag_type': 4549, 'hashtag_proportion': 'sac,textiles,verre,organique,dasri,masque,encombrant,autre_emballage,autre_non_emballage,film_plastique,sac_noir,sac_transparent', 'hashtag_parmi': 'entrant', 'hashtag_weights': {'sac': 6.0, 'textiles': 3.0, 'verre': 3.0, 'organique': 10.0, 'dasri': 10.0, 'masque': 3.0, 'encombrant': 10.0, 'autre_emballage': 3.0, 'autre_non_emballage': 3.0, 'entrant': 1.0, 'sac_noir': 1.0, 'sac_transparent': 1.0, 'film_plastique': 1.0}} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier Fonction SLA_MENSUEL nb_day : (5, 31) VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4346 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4346 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4346 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4346 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) DONE and to test : checkNoCycle ! We are managing only one step so we do not consider checkConsistencyNbInputNbOutput ! We are managing only one step so we do not consider checkConsistencyTypeOutputInput ! {"input_values":[]} SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4510 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4510 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4510 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4510 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! WARNING : number of outputs for step 12695 mask_detect is not consistent : 4 used against 2 in the step definition ! WARNING : number of outputs for step 12698 rle_unique_nms_with_priority is not consistent : 2 used against 1 in the step definition ! Step 12702 crop_condition have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! WARNING : number of outputs for step 12702 crop_condition is not consistent : 4 used against 3 in the step definition ! WARNING : number of outputs for step 12699 ventilate_hashtags_in_portfolio is not consistent : 2 used against 1 in the step definition ! Step 12700 final have less inputs used (2) than in the step definition (3) : maybe we manage optionnal inputs ! Step 12700 final have less outputs used (1) than in the step definition (2) : some outputs may be not used ! Step 12704 split_time_score have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 2 of step 12701 have datatype=6 We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 12699 doesn't seem to be define in the database( WARNING : type of input 3 of step 12700 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 0 of step 12703 have datatype=18 WARNING : type of input 5 of step 12701 doesn't seem to be define in the database( WARNING : output 0 of step 12703 have datatype=11 whereas input 5 of step 12701 have datatype=None WARNING : output 0 of step 12701 have datatype=10 whereas input 0 of step 12704 have datatype=18 WARNING : type of input 2 of step 12702 doesn't seem to be define in the database( WARNING : output 1 of step 12698 have datatype=7 whereas input 2 of step 12702 have datatype=None WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12697 doesn't seem to be define in the database( WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12696 doesn't seem to be define in the database( WARNING : type of output 3 of step 12702 doesn't seem to be define in the database( WARNING : type of input 1 of step 12699 doesn't seem to be define in the database( DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'name_pipeline': 'aggregate_carac_ratio', 'hashtag_type': 4549, 'hashtag_proportion': 'sac,textiles,verre,organique,dasri,masque,encombrant,autre_emballage,autre_non_emballage,film_plastique,sac_noir,sac_transparent', 'hashtag_parmi': 'entrant', 'hashtag_weights': {'sac': 6.0, 'textiles': 3.0, 'verre': 3.0, 'organique': 10.0, 'dasri': 10.0, 'masque': 3.0, 'encombrant': 10.0, 'autre_emballage': 3.0, 'autre_non_emballage': 3.0, 'entrant': 1.0, 'sac_noir': 1.0, 'sac_transparent': 1.0, 'film_plastique': 1.0}} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier TODO : Insert select and so on SELECT id, name, misc_info, datou_id_for_right FROM MTRPhoto.dashboard_places WHERE name="cedre"; SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4346 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4346 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4346 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4346 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) DONE and to test : checkNoCycle ! We are managing only one step so we do not consider checkConsistencyNbInputNbOutput ! We are managing only one step so we do not consider checkConsistencyTypeOutputInput ! {"input_values":[]} SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4510 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4510 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4510 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4510 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! WARNING : number of outputs for step 12695 mask_detect is not consistent : 4 used against 2 in the step definition ! WARNING : number of outputs for step 12698 rle_unique_nms_with_priority is not consistent : 2 used against 1 in the step definition ! Step 12702 crop_condition have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! WARNING : number of outputs for step 12702 crop_condition is not consistent : 4 used against 3 in the step definition ! WARNING : number of outputs for step 12699 ventilate_hashtags_in_portfolio is not consistent : 2 used against 1 in the step definition ! Step 12700 final have less inputs used (2) than in the step definition (3) : maybe we manage optionnal inputs ! Step 12700 final have less outputs used (1) than in the step definition (2) : some outputs may be not used ! Step 12704 split_time_score have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 2 of step 12701 have datatype=6 We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 12699 doesn't seem to be define in the database( WARNING : type of input 3 of step 12700 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 0 of step 12703 have datatype=18 WARNING : type of input 5 of step 12701 doesn't seem to be define in the database( WARNING : output 0 of step 12703 have datatype=11 whereas input 5 of step 12701 have datatype=None WARNING : output 0 of step 12701 have datatype=10 whereas input 0 of step 12704 have datatype=18 WARNING : type of input 2 of step 12702 doesn't seem to be define in the database( WARNING : output 1 of step 12698 have datatype=7 whereas input 2 of step 12702 have datatype=None WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12697 doesn't seem to be define in the database( WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12696 doesn't seem to be define in the database( WARNING : type of output 3 of step 12702 doesn't seem to be define in the database( WARNING : type of input 1 of step 12699 doesn't seem to be define in the database( DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'name_pipeline': 'aggregate_carac_ratio', 'hashtag_type': 4549, 'hashtag_proportion': 'sac,textiles,verre,organique,dasri,masque,encombrant,autre_emballage,autre_non_emballage,film_plastique,sac_noir,sac_transparent', 'hashtag_parmi': 'entrant', 'hashtag_weights': {'sac': 6.0, 'textiles': 3.0, 'verre': 3.0, 'organique': 10.0, 'dasri': 10.0, 'masque': 3.0, 'encombrant': 10.0, 'autre_emballage': 3.0, 'autre_non_emballage': 3.0, 'entrant': 1.0, 'sac_noir': 1.0, 'sac_transparent': 1.0, 'film_plastique': 1.0}} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier place_id : 161 month : 03 mtr_id : 1080 dashboard_places : cedre date : 2025-03 year : 2025 raspi : 20090 select count(distinct mtr_photo_id) from MTRUser.mtr_portfolio_photos where mtr_portfolio_id in (select id from MTRUser.mtr_portfolios where id in (select mtr_portfolio_id from MTRPhoto.dashboard_results where dashboard_run_id in(select last_run_id from MTRPhoto.dashboard_entry_day where dashboard_place_id in (select id from MTRPhoto.dashboard_places where name = 'cedre' and date like '%2025-03%') and created_at like '%2025-03%'))); nb_day : (5, 31) after unwanted_material_data nb_day : (5, 31) WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used WARNING: No hour configured for port 20090, from 00:00 to 24:00 used SQL: SELECT ms.id, ms.name, mst.name as type, ms.test_group, ms.test_name, ms.server, ms.version_code, ms.result_str, ms.result_bool, ms.result_float, ms.lien, ms.created_at FROM MTRAdmin.monitor_sys ms INNER JOIN MTRAdmin.monitor_sys_type mst on mst.id = ms.type WHERE ms.test_name like '%20090%' AND ms.result_str like '%2025/03%'ORDER BY ms.id desc LIMIT 0, 1000000; Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! Missing day sla_per_day needs to be called ! We will need to call the sla_per_day for these missing days : 2025-03-01 00:00:00,2025-03-02 00:00:00,2025-03-03 00:00:00,2025-03-04 00:00:00,2025-03-05 00:00:00,2025-03-06 00:00:00,2025-03-07 00:00:00,2025-03-08 00:00:00,2025-03-09 00:00:00,2025-03-10 00:00:00,2025-03-11 00:00:00,2025-03-12 00:00:00,2025-03-13 00:00:00,2025-03-14 00:00:00,2025-03-15 00:00:00,2025-03-16 00:00:00,2025-03-17 00:00:00,2025-03-18 00:00:00,2025-03-19 00:00:00,2025-03-20 00:00:00,2025-03-21 00:00:00,2025-03-22 00:00:00,2025-03-23 00:00:00,2025-03-24 00:00:00,2025-03-25 00:00:00,2025-03-26 00:00:00,2025-03-27 00:00:00,2025-03-28 00:00:00,2025-03-29 00:00:00,2025-03-30 00:00:00,2025-03-31 00:00:00 after coverage_data SELECT DATE(date_debut) as date, sum(qualite*nombre_balle)/sum(nombre_balle), count(distinct mtr_photo_id) FROM MTRPhoto.dashboard_results dr, MTRUser.mtr_portfolio_photos mpp, MTRPhoto.dashboard_entry_day ded where mpp.mtr_portfolio_id=dr.mtr_portfolio_id and ded.dashboard_place_id = 161 and month(ded.date) = 03 and year(ded.date) = 2025 and dr.dashboard_run_id=ded.last_run_id and dr.qualite>=0 GROUP BY ded.date order by dr.date_debut; SELECT DATE(date_debut) as date, sum(qualite*nombre_balle)/sum(nombre_balle), count(distinct mtr_photo_id) FROM MTRPhoto.dashboard_results dr, MTRUser.mtr_portfolio_photos mpp, MTRPhoto.dashboard_entry_day ded where mpp.mtr_portfolio_id=dr.mtr_portfolio_id and dr.hashtag = '_______entrant' and ded.dashboard_place_id = 161 and month(ded.date) = 03 and year(ded.date) = 2025 and dr.dashboard_run_id=ded.last_run_id and dr.qualite>=0 GROUP BY ded.date order by dr.date_debut; select avg(cps.score) as score, SUBSTRING_INDEX(SUBSTRING_INDEX(mp.name,'__', -1), '_', 1) as sbstr, SUBSTRING_INDEX(SUBSTRING_INDEX(mp.name,'_', -2), '_', 1) as sbstr2, SUBSTRING_INDEX(mp.name,'_', -1) as sbstr3 FROM MTRUser.mtr_portfolio_photos mpp, MTRUser.mtr_portfolios mp,MTRPhoto.class_photo_score cps,MTRBack.hashtags h,MTRPhoto.dashboard_results dr,MTRPhoto.dashboard_entry_day ded, MTRPhoto.dashboard_places dp WHERE mp.id=mpp.mtr_portfolio_id and cps.photo_id=mpp.mtr_photo_id and h.hashtag_id=cps.hashtag_id and dr.mtr_portfolio_id = mpp.mtr_portfolio_id and dr.dashboard_run_id = ded.last_run_id and ded.dashboard_place_id = dp.id and h.hashtag in ('blur', 'mean_blur', 'clear') and dp.id = 161 and mp.name like '%032025%' and cps.thcl = 1533 GROUP BY sbstr ; after number_of_batch date_start : 2025-03-01 : dt_date_just_month_year : 2025-03-01 00:00:00 : VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! after pl.get_datou_sts_from_crontab : verbose : True no sts found, try to find from database SELECT dri.id FROM MTRPhoto.dashboard_run_ids dri, MTRPhoto.dashboard_entry_day ded, MTRPhoto.dashboard_places dp WHERE dp.name= "cedre" AND ded.dashboard_place_id=dp.id AND dri.dashboard_entry_day=ded.id AND dri.id=ded.last_run_id AND ded.date >= "2025-03-01" AND ded.date <= "2025-03-31" apple3 {'entrant': {'mat': 'entrant', 'pht': 4549, 'datou_carac_id': 4510, 'unwanted_material': [], 'hashtag_majoritaire_from_carac': 'entrant'}} SELECT h.hashtag as unwanted_material, substr(dr.hashtag,8) as main_material, ptp.type as pht_type, sum(pcr.value*dr.nombre_balle)/sum(dr.nombre_balle) as ratio, count(distinct mpp.mtr_photo_id) as nb_photo, group_concat(distinct ptp.mtr_portfolio_id_2) as list_port_cont, group_concat(distinct concat(cast(ptp.mtr_portfolio_id_1 as char), ":", cast(ptp.mtr_portfolio_id_2 as char))) as assoc_port, group_concat(distinct concat(cast(ptp.mtr_portfolio_id_1 as char), ":", h.hashtag, ":", cast(ptp.type as char), ":", cast(ptp.mtr_portfolio_id_2 as char))) as assoc_mat FROM MTRPhoto.dashboard_results dr, MTRPhoto.mtr_port_to_port_ids ptp, MTRUser.mtr_portfolio_photos mpp, MTRUser.portfolio_carac_ratio pcr, MTRBack.hashtags h WHERE dr.dashboard_run_id IN (1641282,1643819,1648928,1665818,1662484,1670534,1674867,1662480,1668621,1674857,1674846,1673005,1676817,1679503,1679948) AND dr.mtr_portfolio_id=ptp.mtr_portfolio_id_1 AND dr.qualite >= 0 AND mpp.mtr_portfolio_id=ptp.mtr_portfolio_id_2 AND pcr.portfolio_id=ptp.mtr_portfolio_id_1 AND h.hashtag_id = pcr.hashtag_id AND ptp.type = pcr.hashtag_type AND mpp.hide_status = 0 AND ptp.hashtag_id=h.hashtag_id group by h.hashtag, dr.hashtag, ptp.type; VR TODO TO BETTER PARSE ! ({'unwanted_material': 'autre_emballage', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.01053128742348665, 'nb_photo': 3, 'list_port_cont': '21388421,21405492', 'assoc_port': '21387747:21388421,21405454:21405492', 'assoc_mat': '21387747:autre_emballage:4549:21388421,21405454:autre_emballage:4549:21405492'}, {'unwanted_material': 'film_plastique', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.0014969606590157323, 'nb_photo': 2, 'list_port_cont': '21388417', 'assoc_port': '21387747:21388417', 'assoc_mat': '21387747:film_plastique:4549:21388417'}, {'unwanted_material': 'sac_noir', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.0046107200195015085, 'nb_photo': 3, 'list_port_cont': '21388410', 'assoc_port': '21387747:21388410', 'assoc_mat': '21387747:sac_noir:4549:21388410'}, {'unwanted_material': 'sac_transparent', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.00200866447582867, 'nb_photo': 3, 'list_port_cont': '21388424,21405484', 'assoc_port': '21387747:21388424,21405454:21405484', 'assoc_mat': '21387747:sac_transparent:4549:21388424,21405454:sac_transparent:4549:21405484'}) select count(distinct mtr_photo_id) from MTRUser.mtr_portfolio_photos where mtr_portfolio_id in (select mtr_portfolio_id from MTRPhoto.dashboard_results where dashboard_run_id in(1641282,1643819,1648928,1665818,1662484,1670534,1674867,1662480,1668621,1674857,1674846,1673005,1676817,1679503,1679948)); after get_hostname_from_raspi hasthag : entrant hasthag that could be used but not yet : _______entrant SELECT substr(hashtag,8), substr(date_debut,1,7) as month, (sum(qualite*nombre_balle)/sum(nombre_balle)) FROM MTRPhoto.dashboard_results WHERE hashtag = '_______entrant' and mtr_portfolio_id in (select id FROM MTRUser.mtr_portfolios where mtr_user_id=1080) and qualite >=0 and dashboard_run_id in (select last_run_id from MTRPhoto.dashboard_entry_day where dashboard_place_id=161 and date >='2025-03-01' and date <= '2025-03-31') GROUP BY month order by month; after impurety_average_per_hashtag {'number_of_analysed_photos': ((24,),), 'unwanted_material_monthly_percentage': ((0.014256160974640178,),), 'material_repartition': (('entrant', 24),), 'material_dict': {'entrant': 24}, 'coverage_data': (), 'impurety_average_data': ((datetime.date(2025, 3, 11), 0.0011946424010530479, 10), (datetime.date(2025, 3, 13), 0.02358581709863098, 14)), 'impurety_average_data_per_hashtag': {'entrant': ((datetime.date(2025, 3, 11), 0.0011946424010530479, 10), (datetime.date(2025, 3, 13), 0.02358581709863098, 14))}, 'quality_array': [['1313.3593318914977', '11032025'], ['1269.0034095553217', '13032025']], 'number_of_batch': ((2,),), 'datou_id_sts': 4346, 'map_hashtag_number_mat': {'hash_to_mat_nb': ({'unwanted_material': 'autre_emballage', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.01053128742348665, 'nb_photo': 3, 'list_port_cont': '21388421,21405492', 'assoc_port': '21387747:21388421,21405454:21405492', 'assoc_mat': '21387747:autre_emballage:4549:21388421,21405454:autre_emballage:4549:21405492'}, {'unwanted_material': 'film_plastique', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.0014969606590157323, 'nb_photo': 2, 'list_port_cont': '21388417', 'assoc_port': '21387747:21388417', 'assoc_mat': '21387747:film_plastique:4549:21388417'}, {'unwanted_material': 'sac_noir', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.0046107200195015085, 'nb_photo': 3, 'list_port_cont': '21388410', 'assoc_port': '21387747:21388410', 'assoc_mat': '21387747:sac_noir:4549:21388410'}, {'unwanted_material': 'sac_transparent', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.00200866447582867, 'nb_photo': 3, 'list_port_cont': '21388424,21405484', 'assoc_port': '21387747:21388424,21405454:21405484', 'assoc_mat': '21387747:sac_transparent:4549:21388424,21405454:sac_transparent:4549:21405484'}), 'main_stat': {'nb_photos': 24}}, 'hostname': (('raspi-90',),), 'impurety_average_per_hashtag': [(('entrant', '2025-03', 0.014256160974640178),)]} VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4346 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4346 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4346 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4346 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) DONE and to test : checkNoCycle ! We are managing only one step so we do not consider checkConsistencyNbInputNbOutput ! We are managing only one step so we do not consider checkConsistencyTypeOutputInput ! {"input_values":[]} SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4510 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4510 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4510 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4510 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! WARNING : number of outputs for step 12695 mask_detect is not consistent : 4 used against 2 in the step definition ! WARNING : number of outputs for step 12698 rle_unique_nms_with_priority is not consistent : 2 used against 1 in the step definition ! Step 12702 crop_condition have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! WARNING : number of outputs for step 12702 crop_condition is not consistent : 4 used against 3 in the step definition ! WARNING : number of outputs for step 12699 ventilate_hashtags_in_portfolio is not consistent : 2 used against 1 in the step definition ! Step 12700 final have less inputs used (2) than in the step definition (3) : maybe we manage optionnal inputs ! Step 12700 final have less outputs used (1) than in the step definition (2) : some outputs may be not used ! Step 12704 split_time_score have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 2 of step 12701 have datatype=6 We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 12699 doesn't seem to be define in the database( WARNING : type of input 3 of step 12700 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 0 of step 12703 have datatype=18 WARNING : type of input 5 of step 12701 doesn't seem to be define in the database( WARNING : output 0 of step 12703 have datatype=11 whereas input 5 of step 12701 have datatype=None WARNING : output 0 of step 12701 have datatype=10 whereas input 0 of step 12704 have datatype=18 WARNING : type of input 2 of step 12702 doesn't seem to be define in the database( WARNING : output 1 of step 12698 have datatype=7 whereas input 2 of step 12702 have datatype=None WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12697 doesn't seem to be define in the database( WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12696 doesn't seem to be define in the database( WARNING : type of output 3 of step 12702 doesn't seem to be define in the database( WARNING : type of input 1 of step 12699 doesn't seem to be define in the database( DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'name_pipeline': 'aggregate_carac_ratio', 'hashtag_type': 4549, 'hashtag_proportion': 'sac,textiles,verre,organique,dasri,masque,encombrant,autre_emballage,autre_non_emballage,film_plastique,sac_noir,sac_transparent', 'hashtag_parmi': 'entrant', 'hashtag_weights': {'sac': 6.0, 'textiles': 3.0, 'verre': 3.0, 'organique': 10.0, 'dasri': 10.0, 'masque': 3.0, 'encombrant': 10.0, 'autre_emballage': 3.0, 'autre_non_emballage': 3.0, 'entrant': 1.0, 'sac_noir': 1.0, 'sac_transparent': 1.0, 'film_plastique': 1.0}} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier TODO : Insert select and so on SELECT id, name, misc_info, datou_id_for_right FROM MTRPhoto.dashboard_places WHERE name="cedre"; SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4346 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4346 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4346 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4346 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) DONE and to test : checkNoCycle ! We are managing only one step so we do not consider checkConsistencyNbInputNbOutput ! We are managing only one step so we do not consider checkConsistencyTypeOutputInput ! {"input_values":[]} SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4510 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4510 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4510 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4510 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! WARNING : number of outputs for step 12695 mask_detect is not consistent : 4 used against 2 in the step definition ! WARNING : number of outputs for step 12698 rle_unique_nms_with_priority is not consistent : 2 used against 1 in the step definition ! Step 12702 crop_condition have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! WARNING : number of outputs for step 12702 crop_condition is not consistent : 4 used against 3 in the step definition ! WARNING : number of outputs for step 12699 ventilate_hashtags_in_portfolio is not consistent : 2 used against 1 in the step definition ! Step 12700 final have less inputs used (2) than in the step definition (3) : maybe we manage optionnal inputs ! Step 12700 final have less outputs used (1) than in the step definition (2) : some outputs may be not used ! Step 12704 split_time_score have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 2 of step 12701 have datatype=6 We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 12699 doesn't seem to be define in the database( WARNING : type of input 3 of step 12700 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 12699 have datatype=10 whereas input 0 of step 12703 have datatype=18 WARNING : type of input 5 of step 12701 doesn't seem to be define in the database( WARNING : output 0 of step 12703 have datatype=11 whereas input 5 of step 12701 have datatype=None WARNING : output 0 of step 12701 have datatype=10 whereas input 0 of step 12704 have datatype=18 WARNING : type of input 2 of step 12702 doesn't seem to be define in the database( WARNING : output 1 of step 12698 have datatype=7 whereas input 2 of step 12702 have datatype=None WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12697 doesn't seem to be define in the database( WARNING : type of output 3 of step 12695 doesn't seem to be define in the database( WARNING : type of input 1 of step 12696 doesn't seem to be define in the database( WARNING : type of output 3 of step 12702 doesn't seem to be define in the database( WARNING : type of input 1 of step 12699 doesn't seem to be define in the database( DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'name_pipeline': 'aggregate_carac_ratio', 'hashtag_type': 4549, 'hashtag_proportion': 'sac,textiles,verre,organique,dasri,masque,encombrant,autre_emballage,autre_non_emballage,film_plastique,sac_noir,sac_transparent', 'hashtag_parmi': 'entrant', 'hashtag_weights': {'sac': 6.0, 'textiles': 3.0, 'verre': 3.0, 'organique': 10.0, 'dasri': 10.0, 'masque': 3.0, 'encombrant': 10.0, 'autre_emballage': 3.0, 'autre_non_emballage': 3.0, 'entrant': 1.0, 'sac_noir': 1.0, 'sac_transparent': 1.0, 'film_plastique': 1.0}} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier {'number_of_analysed_photos': ((24,),), 'unwanted_material_monthly_percentage': ((0.014256160974640178,),), 'material_repartition': (('entrant', 24),), 'material_dict': {'entrant': 24}, 'coverage_data': (), 'impurety_average_data': ((datetime.date(2025, 3, 11), 0.0011946424010530479, 10), (datetime.date(2025, 3, 13), 0.02358581709863098, 14)), 'impurety_average_data_per_hashtag': {'entrant': ((datetime.date(2025, 3, 11), 0.0011946424010530479, 10), (datetime.date(2025, 3, 13), 0.02358581709863098, 14))}, 'quality_array': [['1313.3593318914977', '11032025'], ['1269.0034095553217', '13032025']], 'number_of_batch': ((2,),), 'datou_id_sts': 4346, 'map_hashtag_number_mat': {'hash_to_mat_nb': ({'unwanted_material': 'autre_emballage', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.01053128742348665, 'nb_photo': 3, 'list_port_cont': '21388421,21405492', 'assoc_port': '21387747:21388421,21405454:21405492', 'assoc_mat': '21387747:autre_emballage:4549:21388421,21405454:autre_emballage:4549:21405492'}, {'unwanted_material': 'film_plastique', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.0014969606590157323, 'nb_photo': 2, 'list_port_cont': '21388417', 'assoc_port': '21387747:21388417', 'assoc_mat': '21387747:film_plastique:4549:21388417'}, {'unwanted_material': 'sac_noir', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.0046107200195015085, 'nb_photo': 3, 'list_port_cont': '21388410', 'assoc_port': '21387747:21388410', 'assoc_mat': '21387747:sac_noir:4549:21388410'}, {'unwanted_material': 'sac_transparent', 'main_material': 'entrant', 'pht_type': 4549, 'ratio': 0.00200866447582867, 'nb_photo': 3, 'list_port_cont': '21388424,21405484', 'assoc_port': '21387747:21388424,21405454:21405484', 'assoc_mat': '21387747:sac_transparent:4549:21388424,21405454:sac_transparent:4549:21405484'}), 'main_stat': {'nb_photos': 24}}, 'hostname': (('raspi-90',),), 'impurety_average_per_hashtag': [(('entrant', '2025-03', 0.014256160974640178),)]} filepath : /data_2/data_log/job/memo/cedre/sla_mensuel_cedre_mois_03_annee_2025.pdf Before plot_photos_and_unwanted_material_summary # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) DONE and to test : checkNoCycle ! We are managing only one step so we do not consider checkConsistencyNbInputNbOutput ! We are managing only one step so we do not consider checkConsistencyTypeOutputInput ! None was removed should we ? donnée sous forme de texte was removed should we ? [ptf_id0,ptf_id1...] was removed should we ? Before organize_and_plot_unwanted_material_stats {'entrant': {'autre_emballage': ['autre_emballage', '1.05%', 3], 'film_plastique': ['film_plastique', '0.15%', 2], 'sac_noir': ['sac_noir', '0.46%', 3], 'sac_transparent': ['sac_transparent', '0.2%', 3]}} before organize_and_plot_impurety_average_data before organize_and_plot_quality_photo_data finish generating filepath : /data_2/data_log/job/memo/cedre/sla_mensuel_cedre_mois_03_annee_2025.pdf le fichier se trouve ici : /data_2/data_log/job/memo/cedre 2.68user 2.52system 0:05.40elapsed 96%CPU (0avgtext+0avgdata 190832maxresident)k 432inputs+176outputs (2major+68074minor)pagefaults 0swaps