FBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFwe have missing 0 photos in the step downloads : photo missing : [] try to delete the photos missing in DB length of list_filenames : 200 ; length of list_pids : 200 ; length of list_args : 200 time to download the photos : 23.504334688186646 About to test input to load we should then remove the video here, and this would fix the bug of datou_current ! Calling datou_exec Inside datou_exec : verbose : 0 number of steps : 2 step1:tfhub_classification2 Wed Apr 9 10:30:52 2025 VR 17-11-17 : now, only for linear exec dependencies tree, some output goes to fill the input of the next VR 22-3-18 : now we test the dependencies tree, but keep two separate code for datou_prepare_output_input until the code is correctly tested, clean and works in both case VR 22-3-18 : but we use the first code for the first step id = -1, build in the code of datou_exec VR 22-3-18 : we should manage here the case when we are at the first step instead of building this step before datou_exec Beginning of datou_step TFHub with tf2 ! nombre de thcls : 3 we are using the classfication for multi_thcl [3513, 3379, 3847] begin to check gpu status inside check gpu memory 2025-04-09 10:30:55.533982: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2025-04-09 10:30:55.566845: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1561] Found device 0 with properties: pciBusID: 0000:41:00.0 name: NVIDIA GeForce RTX 2080 Ti computeCapability: 7.5 coreClock: 1.545GHz coreCount: 68 deviceMemorySize: 10.76GiB deviceMemoryBandwidth: 573.69GiB/s 2025-04-09 10:30:55.567226: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2025-04-09 10:30:55.569536: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2025-04-09 10:30:55.571838: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2025-04-09 10:30:55.572194: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2025-04-09 10:30:55.574547: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2025-04-09 10:30:55.575716: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2025-04-09 10:30:55.580756: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2025-04-09 10:30:55.582170: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1703] Adding visible gpu devices: 0 2025-04-09 10:30:55.582916: I tensorflow/core/platform/cpu_feature_guard.cc:143] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA 2025-04-09 10:30:55.611092: I tensorflow/core/platform/profile_utils/cpu_utils.cc:102] CPU Frequency: 3493065000 Hz 2025-04-09 10:30:55.613118: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x7f6b5c000b60 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2025-04-09 10:30:55.613175: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2025-04-09 10:30:55.768100: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x27fdf410 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2025-04-09 10:30:55.768146: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): NVIDIA GeForce RTX 2080 Ti, Compute Capability 7.5 2025-04-09 10:30:55.768879: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1561] Found device 0 with properties: pciBusID: 0000:41:00.0 name: NVIDIA GeForce RTX 2080 Ti computeCapability: 7.5 coreClock: 1.545GHz coreCount: 68 deviceMemorySize: 10.76GiB deviceMemoryBandwidth: 573.69GiB/s 2025-04-09 10:30:55.769012: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2025-04-09 10:30:55.769036: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2025-04-09 10:30:55.769059: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2025-04-09 10:30:55.769081: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2025-04-09 10:30:55.769103: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2025-04-09 10:30:55.769124: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2025-04-09 10:30:55.769146: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2025-04-09 10:30:55.770256: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1703] Adding visible gpu devices: 0 2025-04-09 10:30:55.770315: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2025-04-09 10:30:55.771021: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] Device interconnect StreamExecutor with strength 1 edge matrix: 2025-04-09 10:30:55.771041: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1108] 0 2025-04-09 10:30:55.771052: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1121] 0: N 2025-04-09 10:30:55.772212: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1247] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 3096 MB memory) -> physical GPU (device: 0, name: NVIDIA GeForce RTX 2080 Ti, pci bus id: 0000:41:00.0, compute capability: 7.5) l 3637 free memory gpu now : 4063 max_wait_temp : 1 max_wait : 5 1 Physical GPUs, 1 Logical GPUs tagging for thcl : 3513 To do loadFromThcl(), then load ParamDescType : thcl3513 thcls : [{'id': 3513, 'mtr_user_id': 31, 'name': 'Rungis_amount_dechets_fall_2018_v2_tf', 'pb_hashtag_id': 0, 'live': b'\x00', 'list_hashtags': '05102018_Papier_non_papier_dense,05102018_Papier_non_papier_peu_dense,05102018_Papier_non_papier_presque_vide,05102018_Papier_non_papier_tres_dense,05102018_Papier_non_papier_tres_peu_dense', 'svm_portfolios_learning': '1108385,1108386,1108388,1108384,1108387', 'photo_hashtag_type': 4557, 'photo_desc_type': 5767, 'type_classification': 'tf_classification2', 'hashtag_id_list': '2107751013,2107751014,2107751015,2107751016,2107751017'}] thcl {'id': 3513, 'mtr_user_id': 31, 'name': 'Rungis_amount_dechets_fall_2018_v2_tf', 'pb_hashtag_id': 0, 'live': b'\x00', 'list_hashtags': '05102018_Papier_non_papier_dense,05102018_Papier_non_papier_peu_dense,05102018_Papier_non_papier_presque_vide,05102018_Papier_non_papier_tres_dense,05102018_Papier_non_papier_tres_peu_dense', 'svm_portfolios_learning': '1108385,1108386,1108388,1108384,1108387', 'photo_hashtag_type': 4557, 'photo_desc_type': 5767, 'type_classification': 'tf_classification2', 'hashtag_id_list': '2107751013,2107751014,2107751015,2107751016,2107751017'} Update svm_hashtag_type_desc : 5767 FOUND : 1 Here is data_from_sql_as_vec to set the ParamDescriptorType : (5767, 'Rungis_amount_dechets_fall_2018_v2_tf', 2048, 2048, 'Rungis_amount_dechets_fall_2018_v2_tf', 'pool5', 10.0, None, None, 256, None, 0, None, 8, None, None, -1000.0, 3, datetime.datetime(2023, 3, 16, 15, 52, 10), datetime.datetime(2023, 3, 16, 15, 52, 10)) model_name : Rungis_amount_dechets_fall_2018_v2_tf model_param file didn't exist model_name : Rungis_amount_dechets_fall_2018_v2_tf model_type : tf_classification2 list file need : ['Confusion_Matrix.png', 'Precision_Recall_05102018_Papier_non_papier_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_peu_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_presque_vide.jpg', 'Precision_Recall_05102018_Papier_non_papier_tres_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_tres_peu_dense.jpg', 'Result_Summary.txt', 'checkpoint', 'model_checkpoint.ckpt.data-00000-of-00001', 'model_checkpoint.ckpt.data-00000-of-00002', 'model_checkpoint.ckpt.data-00001-of-00002', 'model_checkpoint.ckpt.index', 'model_weights.h5'] file exist in s3 : ['Confusion_Matrix.png', 'Precision_Recall_05102018_Papier_non_papier_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_peu_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_presque_vide.jpg', 'Precision_Recall_05102018_Papier_non_papier_tres_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_tres_peu_dense.jpg', 'Result_Summary.txt', 'checkpoint', 'model_checkpoint.ckpt.data-00000-of-00001', 'model_checkpoint.ckpt.data-00000-of-00002', 'model_checkpoint.ckpt.data-00001-of-00002', 'model_checkpoint.ckpt.index', 'model_weights.h5'] file manque in s3 : [] local folder : /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Confusion_Matrix.png size_local : 67810 size in s3 : 67810 create time local : 2023-10-30 16:21:37 create time in s3 : 2023-10-30 14:09:29 Confusion_Matrix.png already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Precision_Recall_05102018_Papier_non_papier_dense.jpg size_local : 73949 size in s3 : 73949 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:30 Precision_Recall_05102018_Papier_non_papier_dense.jpg already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Precision_Recall_05102018_Papier_non_papier_peu_dense.jpg size_local : 85572 size in s3 : 85572 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:39 Precision_Recall_05102018_Papier_non_papier_peu_dense.jpg already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Precision_Recall_05102018_Papier_non_papier_presque_vide.jpg size_local : 72361 size in s3 : 72361 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:37 Precision_Recall_05102018_Papier_non_papier_presque_vide.jpg already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Precision_Recall_05102018_Papier_non_papier_tres_dense.jpg size_local : 83567 size in s3 : 83567 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:48 Precision_Recall_05102018_Papier_non_papier_tres_dense.jpg already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Precision_Recall_05102018_Papier_non_papier_tres_peu_dense.jpg size_local : 71611 size in s3 : 71611 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:29 Precision_Recall_05102018_Papier_non_papier_tres_peu_dense.jpg already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Result_Summary.txt size_local : 1058 size in s3 : 1058 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:30 Result_Summary.txt already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/checkpoint size_local : 99 size in s3 : 99 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:36 checkpoint already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/model_checkpoint.ckpt.data-00000-of-00001 size_local : 188538519 size in s3 : 188538519 create time local : 2023-10-30 16:21:41 create time in s3 : 2023-10-30 14:09:40 model_checkpoint.ckpt.data-00000-of-00001 already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/model_checkpoint.ckpt.data-00000-of-00002 size_local : 216572 size in s3 : 216572 create time local : 2023-10-30 16:21:41 create time in s3 : 2023-03-16 14:52:09 model_checkpoint.ckpt.data-00000-of-00002 already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/model_checkpoint.ckpt.data-00001-of-00002 size_local : 32279708 size in s3 : 32279708 create time local : 2023-10-30 16:21:42 create time in s3 : 2023-03-16 14:52:07 model_checkpoint.ckpt.data-00001-of-00002 already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/model_checkpoint.ckpt.index size_local : 28001 size in s3 : 28001 create time local : 2023-10-30 16:21:42 create time in s3 : 2023-10-30 14:09:30 model_checkpoint.ckpt.index already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/model_weights.h5 size_local : 94501976 size in s3 : 94501976 create time local : 2023-10-30 16:21:44 create time in s3 : 2023-10-30 14:09:37 model_weights.h5 already exist and didn't need to update desc size : 2048 Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= module (KerasLayer) (None, 2048) 23561152 _________________________________________________________________ Rungis_amount_dechets_fall_2 (None, 5) 10245 ================================================================= Total params: 23,571,397 Trainable params: 10,245 Non-trainable params: 23,561,152 _________________________________________________________________ Loading Weights... time used to create the model : 10.154488563537598 time used to load_weights : 0.2704744338989258 0it [00:00, ?it/s]temp/1744187429_489389_1350716830_7449f925b9f215363f58d4531659e920.jpg temp/1744187429_489389_1350716811_a1e9f65592cf1ece0ec74348ec0a9961.jpg temp/1744187429_489389_1350716785_dc2512332f1c4793b4ec8b14e30b7606.jpg temp/1744187429_489389_1350716724_1f0b7842cf71d68378e3a8842c3ecbaf.jpg temp/1744187429_489389_1350716698_5fab22633040d5a16caef41e007c7056.jpg temp/1744187429_489389_1350716663_7d3daf3577b8935d8a64ec8095f1e3f8.jpg temp/1744187429_489389_1350716222_56f5ad89805e056f21501ac31816e05c.jpg temp/1744187429_489389_1350716149_57e59e718066b26a8f379e50de3f5d30.jpg temp/1744187429_489389_1350716070_1a0fb6fc3fdb9f9f9b6406f512822915.jpg temp/1744187429_489389_1350715989_c90510edb85ded45c81b9cecf2e41921.jpg temp/1744187429_489389_1350715916_3c97ba96e9b78de5e235c389c37ac77b.jpg temp/1744187429_489389_1350715790_78d41639a512930ff4e021f9dec5cfb7.jpg temp/1744187429_489389_1350715473_8e8f37d3363791d5965d2fac42a8cd70.jpg temp/1744187429_489389_1350715357_5d65aa8a1cb97f79523ff6f3a751dd25.jpg temp/1744187429_489389_1350715297_c4f2d76e141c7d78833f84c7acfe3190.jpg temp/1744187429_489389_1350715166_b1afa7a339a556e3134164f86eeb8e75.jpg temp/1744187429_489389_1350715120_cf0fee9ac47a6346ac8070afd7b633cf.jpg temp/1744187429_489389_1350715080_65d58c1e394396a99e2f3b6bc9e6323c.jpg temp/1744187429_489389_1350714711_8de6a317e5b1cbfc5333ff83a634eea4.jpg temp/1744187429_489389_1350714655_4028675c37e3acc33bd97f7ebc1bb790.jpg temp/1744187429_489389_1350714559_437d1197020a3c209fa5e55d3308ccbf.jpg temp/1744187429_489389_1350714493_7c8a5b99fc9901ec5773666f8c66e3a9.jpg temp/1744187429_489389_1350714387_4d277b9367eb9064d436f7462a0acef2.jpg temp/1744187429_489389_1350714323_05c7bfe06d272e8928718100395f7778.jpg temp/1744187429_489389_1350714150_9d44ca86feb7e2353a6c6fd029d4849e.jpg temp/1744187429_489389_1350714034_af95ccb59b0dfe787f74240d09bfca0d.jpg temp/1744187429_489389_1350713950_9db3dccae2d825bda9fa3ab57c085c97.jpg temp/1744187429_489389_1350713870_9104a72feef55484c909bd7db3351e21.jpg temp/1744187429_489389_1350713748_50a68575987bf5a244ae5abf0bbcb6a9.jpg temp/1744187429_489389_1350713652_fca8d4eada2cd1ec0a23d2ceef399a1f.jpg temp/1744187429_489389_1350713575_15ee6792716914478824394436ff538d.jpg temp/1744187429_489389_1350713538_9c8e6167e3ca7c5495e5ee760c240dba.jpg temp/1744187429_489389_1350713485_67dca8c687d5507b27ca11e46c0c38d1.jpg temp/1744187429_489389_1350713402_fc012f7c5e023088e5c47164dbaac420.jpg temp/1744187429_489389_1350713292_86413f2531a16451881563ee5c538039.jpg temp/1744187429_489389_1350713232_e2d596bef0eb8d7ec543ceead449913e.jpg temp/1744187429_489389_1350713079_5476d93a06835ad899862f399a99ea3d.jpg temp/1744187429_489389_1350713016_d8aff539f803969b1f8c78b42f0febd7.jpg temp/1744187429_489389_1350712941_f13a389849de960b47fbf955fee332c7.jpg temp/1744187429_489389_1350712858_3ab446ea8243456eb7856d334414e42e.jpg temp/1744187429_489389_1350712781_9367fb722e3c2759c01adb73b75aa06a.jpg temp/1744187429_489389_1350712690_346369acd6b0ab29c52f50397675d5b1.jpg temp/1744187429_489389_1350712443_9d54f9f9834dacf5753135894f3a4ea4.jpg temp/1744187429_489389_1350712273_a1e2f401c0213a7b2356a5073adf9762.jpg temp/1744187429_489389_1350712229_fd7ce0089a33c8c4c899ccaabe56fd35.jpg temp/1744187429_489389_1350712177_04134c7ef2de66af42da2b2306607f36.jpg temp/1744187429_489389_1350712130_e9dd19a2a3cc5555c41997d230d28b7f.jpg temp/1744187429_489389_1350712099_74300fc6d2f2fba136dd9f5f24d826e7.jpg temp/1744187429_489389_1350711892_52712adbd2fdf2423fe8ae9ef618eb99.jpg temp/1744187429_489389_1350711871_6d8af48f9bd1159222cdf51aa6423b16.jpg temp/1744187429_489389_1350711834_e08cec2e03b5ebb35382c80919996445.jpg temp/1744187429_489389_1350711794_2120f69e7fdd90a07ec2cd2c93246be6.jpg temp/1744187429_489389_1350711746_f75dce1fb6b7b9a0e9327cc3b60967de.jpg temp/1744187429_489389_1350711700_460ae71ef3fde4c1852ce9798e64fa03.jpg temp/1744187429_489389_1350711545_0fa2e98c383764b887c8d1a091641301.jpg temp/1744187429_489389_1350711454_a0b944daf72963576c82a810f41b6c92.jpg temp/1744187429_489389_1350711393_88d49fd9faf67519aa88a0a0da7d002b.jpg temp/1744187429_489389_1350711370_0c382f2332c63ce96ae6d2a52fbfde79.jpg temp/1744187429_489389_1350711295_77a67f4c21c986cb1c7a1d3692dba2b4.jpg temp/1744187429_489389_1350711215_314187387e4f29e807384b6c5cbf092e.jpg temp/1744187429_489389_1350710939_6f143476aaca6b3c1df09a2a309bad1f.jpg temp/1744187429_489389_1350710892_67b4bae9dcab708dfbf884a69d88040c.jpg temp/1744187429_489389_1350710836_658d74ad5a96698f4eebd2d0c023cd6e.jpg temp/1744187429_489389_1350710741_3141de156a265cb9c6fdc772ecd52911.jpg temp/1744187429_489389_1350710649_5edf57cbf0107fe2e8d8cbc0cb25ab41.jpg temp/1744187429_489389_1350710593_a3b99f6c8f722dcc525f86780a97d411.jpg temp/1744187429_489389_1350710293_0195f531784e3839ebf3b3677583934b.jpg temp/1744187429_489389_1350710196_8b4b01dc4602748c67afb73dc410ef55.jpg temp/1744187429_489389_1350710066_ed190e29aa5a2437eaf36a2059f76b8b.jpg temp/1744187429_489389_1350709955_65219a3150525c8cc5a23b93608b0f9c.jpg temp/1744187429_489389_1350709870_fff691c56e808e95926f259483febcc7.jpg temp/1744187429_489389_1350709786_698f63becb242200dba7f43024fd2e0a.jpg temp/1744187429_489389_1350709633_fb7b90229f1543d6094d66a05ba133eb.jpg temp/1744187429_489389_1350709581_f370ac918bc8990f7e43af3f3f9c1106.jpg temp/1744187429_489389_1350709499_7da76367bdadace351ff7a1d2f205027.jpg temp/1744187429_489389_1350709422_ddfec45f2b84572f688deac98d69123b.jpg temp/1744187429_489389_1350709330_edb2946652b7957d7bf501a8210bdf75.jpg temp/1744187429_489389_1350709252_6686ee2d09d8ff02f66ce6fb33afcc38.jpg temp/1744187429_489389_1350709054_7a5527391be8b433d6ebc76ded40f275.jpg temp/1744187429_489389_1350708978_dab57362236ee091da250c316cbd0e95.jpg temp/1744187429_489389_1350708931_a57ec0eabbabd34ce403bc8b37c6c77f.jpg temp/1744187429_489389_1350708831_b42572de792cc99b29366090bcf7e2cf.jpg temp/1744187429_489389_1350708695_4872c86f1ef5af1cd32442d22fa395b3.jpg temp/1744187429_489389_1350708645_d97cb378c78c322bcb05b96c79c62c44.jpg temp/1744187429_489389_1350708388_e6cf5e16e50a43d71891bb636e037e71.jpg temp/1744187429_489389_1350708341_b66e6151899b50a242a2b20bf6ee3a3a.jpg temp/1744187429_489389_1350708266_68ce30af6e6410598b350130eee3d076.jpg temp/1744187429_489389_1350708191_42003c463400ccb65df6d5085a4dcfb9.jpg temp/1744187429_489389_1350708111_1e064f3cbe266e31a33e76f9d1d1dc56.jpg temp/1744187429_489389_1350707883_28bac2414aa09d96b68aba856da093e8.jpg temp/1744187429_489389_1350707366_37559e23085e9f89d1242736214cc7fb.jpg temp/1744187429_489389_1350707325_c714ca2de6698a7cbd56ca45d49ebef8.jpg temp/1744187429_489389_1350707303_0c6e07f15a9c2a9673757260947d7796.jpg temp/1744187429_489389_1350707267_0173fffd6d66f117c006cbbd06f33191.jpg temp/1744187429_489389_1350707236_fd10f4584d4fef501e98bc7a20b3166d.jpg temp/1744187429_489389_1350707187_d121a3627ed00677156b52299d68ccfc.jpg temp/1744187429_489389_1350707075_97b928731ede63e75519cc8122d8725f.jpg temp/1744187429_489389_1350707045_80ee6748464a33949b4dee9999029662.jpg temp/1744187429_489389_1350707031_ba65502142de3cd840fc46cbe39f5863.jpg temp/1744187429_489389_1350706970_ef9379b051512350cbedd0183e175d7d.jpg temp/1744187429_489389_1350706917_515bfc0d851062e67dd2d733d69b76b3.jpg temp/1744187429_489389_1350706876_ddacd185494b8b5b72f9ff103fe44102.jpg temp/1744187429_489389_1350706798_404b0df79ece931726fc122123c0710c.jpg temp/1744187429_489389_1350706772_b448c7fdaeb6a351c598bbb0b2d48f30.jpg temp/1744187429_489389_1350706766_a7315baa0fd7b38471a130bedd0eb5ca.jpg temp/1744187429_489389_1350706760_1d1edcf7f458b19dad727fdab53ff0be.jpg temp/1744187429_489389_1350706722_e8d090421c6586dd5b65820412251693.jpg temp/1744187429_489389_1350706718_e23bab1f41ef6b6336eb8616a7acc60b.jpg temp/1744187429_489389_1350706576_a289aebc4e4aff8b08bd976f9ae669c7.jpg temp/1744187429_489389_1350706529_93d3f64004aaec35d40e35fbef360b4a.jpg temp/1744187429_489389_1350706490_182eb759aee41b574dfee651869a8fc1.jpg temp/1744187429_489389_1350706439_eda9d862e7e575002d39b98237de3551.jpg temp/1744187429_489389_1350706396_02f8d116522eeb0b8db14ff522a75eeb.jpg temp/1744187429_489389_1350706379_48086f8d39f1a4562e85f4e9512d1c11.jpg temp/1744187429_489389_1350693462_106da4d196341899906fcc02d5e782cf.jpg 143it [00:00, 1423.45it/s] 200it [00:00, 1357.28it/s]2025-04-09 10:31:08.809388: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2025-04-09 10:31:09.030265: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2025-04-09 10:31:10.061314: E tensorflow/stream_executor/cuda/cuda_dnn.cc:329] Could not create cudnn handle: CUDNN_STATUS_INTERNAL_ERROR 2025-04-09 10:31:10.075289: E tensorflow/stream_executor/cuda/cuda_dnn.cc:329] Could not create cudnn handle: CUDNN_STATUS_INTERNAL_ERROR temp/1744187429_489389_1350693377_14ee38f4ebed7d113320ef51897d8ee4.jpg temp/1744187429_489389_1350693163_5f4d4413ec56809da53cf1d9147c8f91.jpg temp/1744187429_489389_1350693098_da05a0154b5af87179e75318723d9746.jpg temp/1744187429_489389_1350693048_2acfbe9e7382bef90784f5d5167ceb20.jpg temp/1744187429_489389_1350693003_2ae21c6e705f3b4a1c72bba4052fd500.jpg temp/1744187429_489389_1350692962_986e480b68792f06a3c8c314322f7347.jpg temp/1744187429_489389_1350692926_6cf5901e662511dc5d3b3f8369454082.jpg temp/1744187429_489389_1350692785_7a858410707aa6b71c1b6b87a7161c19.jpg temp/1744187429_489389_1350692756_14c84e804af00c34abca4d371886b2e7.jpg temp/1744187429_489389_1350692707_54c50835cb12a74e32f6f7eac152bff3.jpg temp/1744187429_489389_1350692613_9116d9dfe994faab2fc8efcabc313035.jpg temp/1744187429_489389_1350692562_fa09932565b8d5548b641714e94763e1.jpg temp/1744187429_489389_1350692517_2e3db899001e989650d2f4076ffe58e9.jpg temp/1744187429_489389_1350692176_e0277b6da02fed675ad73a03270510e9.jpg temp/1744187429_489389_1350692140_87e5daa046452945c7d2882bbf1bb5a9.jpg temp/1744187429_489389_1350692094_77c98f26850bd3d1f23b96be6ea9d0ed.jpg temp/1744187429_489389_1350692025_6c741cd06bc20b47146434e5dd97352b.jpg temp/1744187429_489389_1350691996_35955ed6bd740f52ceca781d41d8bb63.jpg temp/1744187429_489389_1350691949_d77aa528f76ece5dfff18a61b5de4163.jpg temp/1744187429_489389_1350691729_fa14ac25a62bd05895f54c9bd35ee792.jpg temp/1744187429_489389_1350691684_fed41e21570a3edeb058c9e1ac3e6b11.jpg temp/1744187429_489389_1350691599_77b94b256dcf007768953f7540df3fa4.jpg temp/1744187429_489389_1350691522_ae0543b1cac6f8924b039344012a9a1f.jpg temp/1744187429_489389_1350691478_34ca16ec2f32e94d2ceca28d5e11a493.jpg temp/1744187429_489389_1350691424_64839e4f77cf7aa6a2539174c7e59a9d.jpg temp/1744187429_489389_1350691298_a50f8fb4827d66325c78afbd508e5000.jpg temp/1744187429_489389_1350691243_9ff861e483fe295c24fa9686e9b2c44d.jpg temp/1744187429_489389_1350691187_28b7dfd85c66c3b71b917dfe83885c10.jpg temp/1744187429_489389_1350691118_af26369cc653d6b2fe9c6f86de26efb9.jpg temp/1744187429_489389_1350691081_70b795b63b4a608131606238b225e24d.jpg temp/1744187429_489389_1350691028_4573da14913963fe5f3dfddaaf2194ea.jpg temp/1744187429_489389_1350690807_861ed6031b7e5946a67f5304e351b78a.jpg temp/1744187429_489389_1350690781_eab502e11d457461acf74597418d7041.jpg temp/1744187429_489389_1350690759_bffd6870336fed0a4d4d54cede49cb50.jpg temp/1744187429_489389_1350690716_6380fb7415a12b53a19da70f5574196a.jpg temp/1744187429_489389_1350690496_f3337b5f77316eb7819bbc9ce1176ad0.jpg temp/1744187429_489389_1350690475_07e5eef80fc00e5f148edd0c934460e2.jpg temp/1744187429_489389_1350690451_a1e0d059ba0d5c3feb30664cd71abce5.jpg temp/1744187429_489389_1350690413_8c7c349d2d695bc45f6c5cf9f6dada45.jpg temp/1744187429_489389_1350690316_954913fa70cd1d074ea3da190ad32a03.jpg temp/1744187429_489389_1350690249_df0919c68133e502bebb304ae1ca885d.jpg temp/1744187429_489389_1350690149_3fb39119ab762cd1e68c4e62c018240d.jpg temp/1744187429_489389_1350690147_0555a9efe3a8312cee4c0bc587a68e65.jpg temp/1744187429_489389_1350690143_d3438098dbf0fae8468b20e811ca365d.jpg temp/1744187429_489389_1350690128_c0f91d21e21ababd518f89ded13fd59b.jpg temp/1744187429_489389_1350690120_e2690b810181c3785a215bab684e53c7.jpg temp/1744187429_489389_1350690106_d9c2f42c6bf70fd9746a21680bc843ac.jpg temp/1744187429_489389_1350690073_9cfe01636ff4cbe63ae9a9ead3d4dfd6.jpg temp/1744187429_489389_1350690072_466a30013b12d6f6e3618417b051ea1d.jpg temp/1744187429_489389_1350690070_c1d242b9ccf0d43034aec88d08c42c9e.jpg temp/1744187429_489389_1350690061_3633cc66872ddd4d32ffeedb862a5a5b.jpg temp/1744187429_489389_1350690057_d58ea4ed8af650a9c844fe105fcf641b.jpg temp/1744187429_489389_1350690054_fc6d0d3812efb2a682850605fd0897d7.jpg temp/1744187429_489389_1350690024_250d298303d7b4eed7fd38a289d619bd.jpg temp/1744187429_489389_1350690009_3f10dd814a8054684282daf4a3862d4f.jpg temp/1744187429_489389_1350690003_dc9fff18cee0979ed71102762eb3ad66.jpg temp/1744187429_489389_1350690000_e0b8c0edd3657a511c918e7932eec193.jpg temp/1744187429_489389_1350689997_961cd6c42aefda581a03f7cf56b0e5ee.jpg temp/1744187429_489389_1350689994_cb1ebd708e6814b4762c2e35b2db8c12.jpg temp/1744187429_489389_1350689986_3747e219cf00af99dd49e6af55e543b0.jpg temp/1744187429_489389_1350689984_76d989ef9232394348e14e088f865c40.jpg temp/1744187429_489389_1350689980_a0259baa2a1e49c0d37dc3e47fda9482.jpg temp/1744187429_489389_1350689976_b08c71e5eb2b8870de7fba84e7405dfd.jpg temp/1744187429_489389_1350689973_7d294a581476ccea609f69f4158a3201.jpg temp/1744187429_489389_1350689971_ae995df3062648509f88b370745f19e8.jpg temp/1744187429_489389_1350689829_01fa9c102277517819c5f5aec168bd93.jpg temp/1744187429_489389_1350689790_48d3433f59d05e3857ac40fa395035c5.jpg temp/1744187429_489389_1350689756_39987fdf9bdb2f2e8dc876f3834f5782.jpg temp/1744187429_489389_1350689722_a44c254c0c60fcbb9de2094b5d2a0755.jpg temp/1744187429_489389_1350689682_fff4d1610425501b17473ebebf1d72a0.jpg temp/1744187429_489389_1350689635_fefa11cf9439cf7a46f8ec1bb75b4423.jpg temp/1744187429_489389_1350689469_76804ae1cdcb3fed8c3a386abbd4b787.jpg temp/1744187429_489389_1350689443_fcbe47195c3fef0a359342cfda5c66a5.jpg temp/1744187429_489389_1350689435_87aa135761423ce687f24ba742be695f.jpg temp/1744187429_489389_1350689427_57499e54b9c0a3ffa9f2239ff5773250.jpg temp/1744187429_489389_1350689420_06693c20727903ca4ca2e804d9f85c53.jpg temp/1744187429_489389_1350689413_cea03fb0eb740af5e71c4c0b591552d6.jpg temp/1744187429_489389_1350675502_a0afa378a3bd0b5240e93e78301200f8.jpg temp/1744187429_489389_1350675498_a5f50ed40c6537153e4791fc5de0258a.jpg temp/1744187429_489389_1350675496_8e3386306e23d5fec010c8f8d1e3dfee.jpg temp/1744187429_489389_1350675493_369c3f7334651a0a27152a0061449ae5.jpg temp/1744187429_489389_1350675488_2f46acc30533b2b30fefa45585a8875e.jpg temp/1744187429_489389_1350675482_fca57e8c2a733a94a149a7ee583c72cf.jpg temp/1744187429_489389_1350675478_18f70ac2fb6093c3bb456ead9c2bc026.jpg temp/1744187429_489389_1350675475_0a8970161ffb13b9844df931ac9c62c7.jpg Found 200 images belonging to 1 classes. begin to do the prediction : ERROR in datou_step_exec, will save and exit ! Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above. [[{{node model/module/StatefulPartitionedCall/StatefulPartitionedCall/StatefulPartitionedCall/conv1/StatefulPartitionedCall/Conv2D}}]] [Op:__inference_predict_function_34490] Function call stack: predict_function File "/home/admin/workarea/git/Velours/python/mtr/datou/datou_lib.py", line 2329, in datou_exec output = datou_step_exec(sNext, args, cache, context, map_info, verbose, mtr_user_id) File "/home/admin/workarea/git/Velours/python/mtr/datou/datou_lib.py", line 2523, in datou_step_exec return lib_process.datou_step_tfhub2(param, json_param, args, cache, context, map_info, verbose) File "/home/admin/workarea/git/Velours/python/mtr/datou/lib_step_exec/lib_step_process.py", line 3146, in datou_step_tfhub2 classes, outputs, features = this_model.predict_image_paths(list_paths, keep_aspect_ratio=keep_aspect_ratio, File "/home/admin/workarea/git/Velours/python/mtr/tfhub2/evaluate.py", line 288, in predict_image_paths Y_pred, F_pred = self.model.predict(valid_generator, validation_steps) File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/training.py", line 88, in _method_wrapper return method(self, *args, **kwargs) File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/training.py", line 1268, in predict tmp_batch_outputs = predict_function(iterator) File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py", line 580, in __call__ result = self._call(*args, **kwds) File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py", line 650, in _call return self._concrete_stateful_fn._filtered_call(canon_args, canon_kwds) # pylint: disable=protected-access File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py", line 1661, in _filtered_call return self._call_flat( File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py", line 1745, in _call_flat return self._build_call_outputs(self._inference_function.call( File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py", line 593, in call outputs = execute.execute( File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/execute.py", line 59, in quick_execute tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name, [1350716830, 1350716811, 1350716785, 1350716724, 1350716698, 1350716663, 1350716222, 1350716149, 1350716070, 1350715989, 1350715916, 1350715790, 1350715473, 1350715357, 1350715297, 1350715166, 1350715120, 1350715080, 1350714711, 1350714655, 1350714559, 1350714493, 1350714387, 1350714323, 1350714150, 1350714034, 1350713950, 1350713870, 1350713748, 1350713652, 1350713575, 1350713538, 1350713485, 1350713402, 1350713292, 1350713232, 1350713079, 1350713016, 1350712941, 1350712858, 1350712781, 1350712690, 1350712443, 1350712273, 1350712229, 1350712177, 1350712130, 1350712099, 1350711892, 1350711871, 1350711834, 1350711794, 1350711746, 1350711700, 1350711545, 1350711454, 1350711393, 1350711370, 1350711295, 1350711215, 1350710939, 1350710892, 1350710836, 1350710741, 1350710649, 1350710593, 1350710293, 1350710196, 1350710066, 1350709955, 1350709870, 1350709786, 1350709633, 1350709581, 1350709499, 1350709422, 1350709330, 1350709252, 1350709054, 1350708978, 1350708931, 1350708831, 1350708695, 1350708645, 1350708388, 1350708341, 1350708266, 1350708191, 1350708111, 1350707883, 1350707366, 1350707325, 1350707303, 1350707267, 1350707236, 1350707187, 1350707075, 1350707045, 1350707031, 1350706970, 1350706917, 1350706876, 1350706798, 1350706772, 1350706766, 1350706760, 1350706722, 1350706718, 1350706576, 1350706529, 1350706490, 1350706439, 1350706396, 1350706379, 1350693462, 1350693377, 1350693163, 1350693098, 1350693048, 1350693003, 1350692962, 1350692926, 1350692785, 1350692756, 1350692707, 1350692613, 1350692562, 1350692517, 1350692176, 1350692140, 1350692094, 1350692025, 1350691996, 1350691949, 1350691729, 1350691684, 1350691599, 1350691522, 1350691478, 1350691424, 1350691298, 1350691243, 1350691187, 1350691118, 1350691081, 1350691028, 1350690807, 1350690781, 1350690759, 1350690716, 1350690496, 1350690475, 1350690451, 1350690413, 1350690316, 1350690249, 1350690149, 1350690147, 1350690143, 1350690128, 1350690120, 1350690106, 1350690073, 1350690072, 1350690070, 1350690061, 1350690057, 1350690054, 1350690024, 1350690009, 1350690003, 1350690000, 1350689997, 1350689994, 1350689986, 1350689984, 1350689980, 1350689976, 1350689973, 1350689971, 1350689829, 1350689790, 1350689756, 1350689722, 1350689682, 1350689635, 1350689469, 1350689443, 1350689435, 1350689427, 1350689420, 1350689413, 1350675502, 1350675498, 1350675496, 1350675493, 1350675488, 1350675482, 1350675478, 1350675475] begin to insert list_values into mtr_datou_result : length of list_values in save_final : 200 time used for this insertion : 0.0871117115020752 save_final ERROR in last step tfhub_classification2, Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above. [[{{node model/module/StatefulPartitionedCall/StatefulPartitionedCall/StatefulPartitionedCall/conv1/StatefulPartitionedCall/Conv2D}}]] [Op:__inference_predict_function_34490] Function call stack: predict_function time spend for datou_step_exec : 17.50179409980774 time spend to save output : 0.10484671592712402 total time spend for step 0 : 17.606640815734863 need to delete datou_research and reload, so keep current state 1 caffe_path_current : About to save ! 2 After save, about to update current ! 19.27user 5.34system 0:48.89elapsed 50%CPU (0avgtext+0avgdata 2447108maxresident)k 278104inputs+177896outputs (242major+998451minor)pagefaults 0swaps