FBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFBFwe have missing 0 photos in the step downloads : photo missing : [] try to delete the photos missing in DB length of list_filenames : 200 ; length of list_pids : 200 ; length of list_args : 200 time to download the photos : 36.524850606918335 About to test input to load we should then remove the video here, and this would fix the bug of datou_current ! Calling datou_exec Inside datou_exec : verbose : 0 number of steps : 2 step1:tfhub_classification2 Tue May 27 18:11:05 2025 VR 17-11-17 : now, only for linear exec dependencies tree, some output goes to fill the input of the next VR 22-3-18 : now we test the dependencies tree, but keep two separate code for datou_prepare_output_input until the code is correctly tested, clean and works in both case VR 22-3-18 : but we use the first code for the first step id = -1, build in the code of datou_exec VR 22-3-18 : we should manage here the case when we are at the first step instead of building this step before datou_exec Beginning of datou_step TFHub with tf2 ! nombre de thcls : 3 we are using the classfication for multi_thcl [3513, 3379, 3847] begin to check gpu status inside check gpu memory inside check gpu memory inside check gpu memory inside check gpu memory inside check gpu memory inside check gpu memory 2025-05-27 18:11:12.904948: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2025-05-27 18:11:12.934018: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1561] Found device 0 with properties: pciBusID: 0000:41:00.0 name: NVIDIA GeForce RTX 2080 Ti computeCapability: 7.5 coreClock: 1.545GHz coreCount: 68 deviceMemorySize: 10.76GiB deviceMemoryBandwidth: 573.69GiB/s 2025-05-27 18:11:12.934284: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2025-05-27 18:11:12.941558: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2025-05-27 18:11:12.943864: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2025-05-27 18:11:12.945101: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2025-05-27 18:11:12.949314: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2025-05-27 18:11:12.950980: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2025-05-27 18:11:12.957939: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2025-05-27 18:11:12.959185: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1703] Adding visible gpu devices: 0 2025-05-27 18:11:12.959597: I tensorflow/core/platform/cpu_feature_guard.cc:143] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA 2025-05-27 18:11:12.991174: I tensorflow/core/platform/profile_utils/cpu_utils.cc:102] CPU Frequency: 3493065000 Hz 2025-05-27 18:11:12.993098: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x7f4bdc000b60 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2025-05-27 18:11:12.993142: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2025-05-27 18:11:13.104530: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x16cc0880 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2025-05-27 18:11:13.104587: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): NVIDIA GeForce RTX 2080 Ti, Compute Capability 7.5 2025-05-27 18:11:13.105578: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1561] Found device 0 with properties: pciBusID: 0000:41:00.0 name: NVIDIA GeForce RTX 2080 Ti computeCapability: 7.5 coreClock: 1.545GHz coreCount: 68 deviceMemorySize: 10.76GiB deviceMemoryBandwidth: 573.69GiB/s 2025-05-27 18:11:13.105652: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2025-05-27 18:11:13.105681: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2025-05-27 18:11:13.105708: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2025-05-27 18:11:13.105734: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2025-05-27 18:11:13.105760: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2025-05-27 18:11:13.105786: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2025-05-27 18:11:13.105812: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2025-05-27 18:11:13.107129: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1703] Adding visible gpu devices: 0 2025-05-27 18:11:13.107194: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2025-05-27 18:11:13.107983: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] Device interconnect StreamExecutor with strength 1 edge matrix: 2025-05-27 18:11:13.108006: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1108] 0 2025-05-27 18:11:13.108020: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1121] 0: N 2025-05-27 18:11:13.109390: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1247] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 3096 MB memory) -> physical GPU (device: 0, name: NVIDIA GeForce RTX 2080 Ti, pci bus id: 0000:41:00.0, compute capability: 7.5) l 3637 free memory gpu now : 2943 max_wait_temp : 6 max_wait : 5 1 Physical GPUs, 1 Logical GPUs tagging for thcl : 3513 To do loadFromThcl(), then load ParamDescType : thcl3513 thcls : [{'id': 3513, 'mtr_user_id': 31, 'name': 'Rungis_amount_dechets_fall_2018_v2_tf', 'pb_hashtag_id': 0, 'live': b'\x00', 'list_hashtags': '05102018_Papier_non_papier_dense,05102018_Papier_non_papier_peu_dense,05102018_Papier_non_papier_presque_vide,05102018_Papier_non_papier_tres_dense,05102018_Papier_non_papier_tres_peu_dense', 'svm_portfolios_learning': '1108385,1108386,1108388,1108384,1108387', 'photo_hashtag_type': 4557, 'photo_desc_type': 5767, 'type_classification': 'tf_classification2', 'hashtag_id_list': '2107751013,2107751014,2107751015,2107751016,2107751017'}] thcl {'id': 3513, 'mtr_user_id': 31, 'name': 'Rungis_amount_dechets_fall_2018_v2_tf', 'pb_hashtag_id': 0, 'live': b'\x00', 'list_hashtags': '05102018_Papier_non_papier_dense,05102018_Papier_non_papier_peu_dense,05102018_Papier_non_papier_presque_vide,05102018_Papier_non_papier_tres_dense,05102018_Papier_non_papier_tres_peu_dense', 'svm_portfolios_learning': '1108385,1108386,1108388,1108384,1108387', 'photo_hashtag_type': 4557, 'photo_desc_type': 5767, 'type_classification': 'tf_classification2', 'hashtag_id_list': '2107751013,2107751014,2107751015,2107751016,2107751017'} Update svm_hashtag_type_desc : 5767 FOUND : 1 Here is data_from_sql_as_vec to set the ParamDescriptorType : (5767, 'Rungis_amount_dechets_fall_2018_v2_tf', 2048, 2048, 'Rungis_amount_dechets_fall_2018_v2_tf', 'pool5', 10.0, None, None, 256, None, 0, None, 8, None, None, -1000.0, 3, datetime.datetime(2023, 3, 16, 15, 52, 10), datetime.datetime(2023, 3, 16, 15, 52, 10)) model_name : Rungis_amount_dechets_fall_2018_v2_tf model_param file didn't exist model_name : Rungis_amount_dechets_fall_2018_v2_tf model_type : tf_classification2 list file need : ['Confusion_Matrix.png', 'Precision_Recall_05102018_Papier_non_papier_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_peu_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_presque_vide.jpg', 'Precision_Recall_05102018_Papier_non_papier_tres_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_tres_peu_dense.jpg', 'Result_Summary.txt', 'checkpoint', 'model_checkpoint.ckpt.data-00000-of-00001', 'model_checkpoint.ckpt.data-00000-of-00002', 'model_checkpoint.ckpt.data-00001-of-00002', 'model_checkpoint.ckpt.index', 'model_weights.h5'] file exist in s3 : ['Confusion_Matrix.png', 'Precision_Recall_05102018_Papier_non_papier_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_peu_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_presque_vide.jpg', 'Precision_Recall_05102018_Papier_non_papier_tres_dense.jpg', 'Precision_Recall_05102018_Papier_non_papier_tres_peu_dense.jpg', 'Result_Summary.txt', 'checkpoint', 'model_checkpoint.ckpt.data-00000-of-00001', 'model_checkpoint.ckpt.data-00000-of-00002', 'model_checkpoint.ckpt.data-00001-of-00002', 'model_checkpoint.ckpt.index', 'model_weights.h5'] file manque in s3 : [] 2025-05-27 18:11:19.600731: I tensorflow/stream_executor/cuda/cuda_driver.cc:763] failed to allocate 3.02G (3246391296 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY: out of memory 2025-05-27 18:11:19.601376: I tensorflow/stream_executor/cuda/cuda_driver.cc:763] failed to allocate 2.72G (2921752064 bytes) from device: CUDA_ERROR_OUT_OF_MEMORY: out of memory local folder : /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Confusion_Matrix.png size_local : 67810 size in s3 : 67810 create time local : 2023-10-30 16:21:37 create time in s3 : 2023-10-30 14:09:29 Confusion_Matrix.png already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Precision_Recall_05102018_Papier_non_papier_dense.jpg size_local : 73949 size in s3 : 73949 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:30 Precision_Recall_05102018_Papier_non_papier_dense.jpg already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Precision_Recall_05102018_Papier_non_papier_peu_dense.jpg size_local : 85572 size in s3 : 85572 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:39 Precision_Recall_05102018_Papier_non_papier_peu_dense.jpg already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Precision_Recall_05102018_Papier_non_papier_presque_vide.jpg size_local : 72361 size in s3 : 72361 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:37 Precision_Recall_05102018_Papier_non_papier_presque_vide.jpg already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Precision_Recall_05102018_Papier_non_papier_tres_dense.jpg size_local : 83567 size in s3 : 83567 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:48 Precision_Recall_05102018_Papier_non_papier_tres_dense.jpg already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Precision_Recall_05102018_Papier_non_papier_tres_peu_dense.jpg size_local : 71611 size in s3 : 71611 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:29 Precision_Recall_05102018_Papier_non_papier_tres_peu_dense.jpg already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/Result_Summary.txt size_local : 1058 size in s3 : 1058 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:30 Result_Summary.txt already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/checkpoint size_local : 99 size in s3 : 99 create time local : 2023-10-30 16:21:38 create time in s3 : 2023-10-30 14:09:36 checkpoint already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/model_checkpoint.ckpt.data-00000-of-00001 size_local : 188538519 size in s3 : 188538519 create time local : 2023-10-30 16:21:41 create time in s3 : 2023-10-30 14:09:40 model_checkpoint.ckpt.data-00000-of-00001 already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/model_checkpoint.ckpt.data-00000-of-00002 size_local : 216572 size in s3 : 216572 create time local : 2023-10-30 16:21:41 create time in s3 : 2023-03-16 14:52:09 model_checkpoint.ckpt.data-00000-of-00002 already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/model_checkpoint.ckpt.data-00001-of-00002 size_local : 32279708 size in s3 : 32279708 create time local : 2023-10-30 16:21:42 create time in s3 : 2023-03-16 14:52:07 model_checkpoint.ckpt.data-00001-of-00002 already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/model_checkpoint.ckpt.index size_local : 28001 size in s3 : 28001 create time local : 2023-10-30 16:21:42 create time in s3 : 2023-10-30 14:09:30 model_checkpoint.ckpt.index already exist and didn't need to update /data/models_weight/Rungis_amount_dechets_fall_2018_v2_tf/model_weights.h5 size_local : 94501976 size in s3 : 94501976 create time local : 2023-10-30 16:21:44 create time in s3 : 2023-10-30 14:09:37 model_weights.h5 already exist and didn't need to update desc size : 2048 Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= module (KerasLayer) (None, 2048) 23561152 _________________________________________________________________ Rungis_amount_dechets_fall_2 (None, 5) 10245 ================================================================= Total params: 23,571,397 Trainable params: 10,245 Non-trainable params: 23,561,152 _________________________________________________________________ Loading Weights... time used to create the model : 7.996064901351929 time used to load_weights : 0.19456243515014648 0it [00:00, ?it/s] 74it [00:00, 610.97it/s]temp/1748362228_1472522_1360999259_0f33099298aea23090f781c3356b99d1.jpg temp/1748362228_1472522_1360999257_e7f3a865d2d517843defcd3791b67827.jpg temp/1748362228_1472522_1360999242_c3eae96aef185d449d1e176b0ab7aec3.jpg temp/1748362228_1472522_1360999239_de72d97d72e30e00245020e6df79881d.jpg temp/1748362228_1472522_1360999237_fffc4a2f39738b4395700cebc6becb2d.jpg temp/1748362228_1472522_1360999236_b8e2997f5ac6ec86c5420d3cc812b7b2.jpg temp/1748362228_1472522_1360999235_56eb6c7f2162eb8e0263a63445656ca0.jpg temp/1748362228_1472522_1360999234_55f39e6203b7142850221e8dc7141cc3.jpg temp/1748362228_1472522_1360999209_a53782565f0b65386760280e9416a66c.jpg temp/1748362228_1472522_1360999207_1c89ae373d5e41150a97c3bdbdbabba2.jpg temp/1748362228_1472522_1360999205_ac2f6552f2dea479a69a197c7ab09b22.jpg temp/1748362228_1472522_1360999203_c63150b954c28bd3c91a61bdc7f13832.jpg temp/1748362228_1472522_1360999202_83b018b9c63077e73f6def5710824718.jpg temp/1748362228_1472522_1360999201_3f0ab961ae454d324c0e82b776b7c6e5.jpg temp/1748362228_1472522_1360999198_dc4639ecbaaa1a2a2ca32d1ea226867f.jpg temp/1748362228_1472522_1360999196_55f7eb4d79c374b65795a84835de54f5.jpg temp/1748362228_1472522_1360999194_37fccd11bdbb5fc093b5a5a35b6212d6.jpg temp/1748362228_1472522_1360999193_94590618b3944c6eb1cb3b021148a07a.jpg temp/1748362228_1472522_1360999192_ae04d8d7c837ee6b8cf7a0d0ec36cf8e.jpg temp/1748362228_1472522_1360999191_076ea493ea13c5e280e05fee46925555.jpg temp/1748362228_1472522_1360999184_eadb6046aac80c258e523b6098a70050.jpg temp/1748362228_1472522_1360999182_d12166f3a5ae1476ac3ed47d4f663871.jpg temp/1748362228_1472522_1360999180_03f71ce78138d4607c597185f198c8a4.jpg temp/1748362228_1472522_1360999178_0c600c4eb5a2f953224c3f75886121c1.jpg temp/1748362228_1472522_1360999175_eaab72599bae51ed746dca2b1a7f4d46.jpg temp/1748362228_1472522_1360999174_db89edbd34a1bb2d6bd2d3ce039ae924.jpg temp/1748362228_1472522_1360999161_6af7e16725e18932d8e8d14defffd3c7.jpg temp/1748362228_1472522_1360999159_b00c69141e86ba94538f0f09628b1226.jpg temp/1748362228_1472522_1360999158_38f3018d60097af0d3051ef4d49d99a5.jpg temp/1748362228_1472522_1360999156_47d7e57008d4c342669f63cc8c6e9275.jpg temp/1748362228_1472522_1360999155_beeaec467d7695aa55c105e990562659.jpg temp/1748362228_1472522_1360999154_fcb1c2e16bf610d19735e762dc19020b.jpg temp/1748362228_1472522_1360999136_59173bd30f4501edadfd84e54fe7b105.jpg temp/1748362228_1472522_1360999133_33f07b5810a5cf92637c7081ffcdfb4e.jpg temp/1748362228_1472522_1360999131_a8f874fbd7f2f5d3e529973dd75a3896.jpg temp/1748362228_1472522_1360999130_c74319376665429da9d1c43f0e7270c3.jpg temp/1748362228_1472522_1360999129_1791e920ed5e949bcb591792d3376a07.jpg temp/1748362228_1472522_1360999127_cc71cea630783d63e5b0866e52e26205.jpg temp/1748362228_1472522_1360999115_8d40e09a44d1a468baf03a84a3b64314.jpg temp/1748362228_1472522_1360999112_63a15c1fa608303b4f71e50d298a63a7.jpg temp/1748362228_1472522_1360999111_31e11ded8ca5753e302d01a459322648.jpg temp/1748362228_1472522_1360999110_2840211098858aa011b2300697379a2d.jpg temp/1748362228_1472522_1360999109_4decf9f9758f6c0d2fbab4eb859c9345.jpg temp/1748362228_1472522_1360999108_d86b9764a116338f635621bc2ade0710.jpg temp/1748362228_1472522_1360999096_daa8f4556c81e5468b785e6c93118bda.jpg temp/1748362228_1472522_1360999094_8f8ea7ce1d4a5a0c35d030d1676108ed.jpg temp/1748362228_1472522_1360999093_9edfd3afee43696f802dfd0f9c943ba8.jpg temp/1748362228_1472522_1360999092_460722594574481ada62588a921f7f43.jpg temp/1748362228_1472522_1360999091_fbd25b3c4823ad2061be55138dd734b2.jpg temp/1748362228_1472522_1360999090_cca2f1f01a079a4f8c72c7d13be6f5b8.jpg temp/1748362228_1472522_1360999076_5928865acd62ea6eae6e1b2db0f7d7a3.jpg temp/1748362228_1472522_1360999073_e7b17abbe864b67c99e1b6423c8a091b.jpg temp/1748362228_1472522_1360999071_c4f3bfb90be39061c1ef625787d8dc60.jpg temp/1748362228_1472522_1360999070_bdc6a012d8ba179083a1440b8da041cb.jpg temp/1748362228_1472522_1360999069_d6d81c8a7eda020b16822db292495edc.jpg temp/1748362228_1472522_1360999068_93135c47b4d931f8609be49ff05ec4f2.jpg temp/1748362228_1472522_1360999056_df8467aafd3054f7ffa71c02d3407d95.jpg temp/1748362228_1472522_1360999054_3271d1b9aa1ba56052504c36254acbde.jpg temp/1748362228_1472522_1360999052_412985dfea6ccc26749327feea2916c0.jpg temp/1748362228_1472522_1360999051_c555f5dc51ab96064d910e857e96da71.jpg temp/1748362228_1472522_1360999050_d590087b42a2a1ebbc8698a395e01a50.jpg temp/1748362228_1472522_1360999048_211f33c355dcdc53dbd7c74259788d0d.jpg temp/1748362228_1472522_1360999021_49b61c2e0aff0fa5e71630b81546dcbd.jpg temp/1748362228_1472522_1360999019_ce463f0f3a9a420fea699696a2843492.jpg temp/1748362228_1472522_1360999016_bfa692ccc54dbc6b4fe7eee34bc86d02.jpg temp/1748362228_1472522_1360999013_a0afac4c7af244439883bc0377b1ddef.jpg temp/1748362228_1472522_1360999012_76a83f725297759f1f068d5e2732cb31.jpg temp/1748362228_1472522_1360999010_b19ac447d70ebce7c810fc954b6341f2.jpg temp/1748362228_1472522_1360999007_c2f6727cfde73542b78f96276465871c.jpg temp/1748362228_1472522_1360999005_1ca3f3a98ba38e43e58275e1f2314b42.jpg temp/1748362228_1472522_1360999004_b01d4b0c15809f31f756a41afbf375bb.jpg temp/1748362228_1472522_1360999003_cb233e1fe908d811a5b18b4fbf92d2a7.jpg temp/1748362228_1472522_1360999002_1d57f8c38d66cc238d532a97cf04ca7a.jpg temp/1748362228_1472522_1360998999_0f2b59ea966458746a7c6a3d1677cac0.jpg temp/1748362228_1472522_1360998987_7ed1e03bfc4a2bb43a47dc8d3f7d00e2.jpg temp/1748362228_1472522_1360998985_7cdf257608e066cbf3c3e548668d3ce5.jpg temp/1748362228_1472522_1360998982_fcf1e5be3b3ef80d4484a246ce9ec3cf.jpg temp/1748362228_1472522_1360998980_94e41629ab71ef62bdf09d7777b13a41.jpg temp/1748362228_1472522_1360998979_5e9f79410c72f325978c125f73e34ca7.jpg temp/1748362228_1472522_1360998977_8a7fb852f394798d5d166a11a3cb3c0e.jpg temp/1748362228_1472522_1360998964_7265f88666f0a6a0448a120691cbf5d7.jpg temp/1748362228_1472522_1360998961_c6b6e77cf356ad21892b24c093f869ff.jpg temp/1748362228_1472522_1360998959_23b81737caf1470eac55f91e0b13b5b7.jpg temp/1748362228_1472522_1360998957_0ae7740405e0200eaa53d3b606434c7e.jpg temp/1748362228_1472522_1360998956_768a0cc370b6ec13bfd7fc2c930be569.jpg temp/1748362228_1472522_1360998954_360905de6cf61db151e8e7b68cc93796.jpg temp/1748362228_1472522_1360998941_dcd4bc27b4dd0740bd69f61dd71d4a34.jpg temp/1748362228_1472522_1360998938_3c38293618aa8b2e7b080fe8f10248a7.jpg temp/1748362228_1472522_1360998935_917d60a2db9a2db06cd90efc6d518e15.jpg temp/1748362228_1472522_1360998931_58991cfda61fade493b2ed81e5763d0d.jpg temp/1748362228_1472522_1360998929_5ae36f68d63948c7d00d5b0fbf01729f.jpg temp/1748362228_1472522_1360998926_ac1499ad633da7ea05a5f0ab59cce77d.jpg temp/1748362228_1472522_1360998922_9c2b8948a90ca0005fe87ac966bfb267.jpg temp/1748362228_1472522_1360998921_fb0acf88df01adc14c5c7c60d1059fa9.jpg temp/1748362228_1472522_1360998920_07b6ac09de5fa36c3b4a5f48f2b9b140.jpg temp/1748362228_1472522_1360998919_398b983f288d1fe08378738f602847d3.jpg temp/1748362228_1472522_1360998918_b5b9c88a5d0aaff2cdffc8210b1b0ac3.jpg temp/1748362228_1472522_1360998917_5dcc069551c4c82f7a274fda1079c2ac.jpg temp/1748362228_1472522_1360998890_8827c7275458ae241285b68bf4e210f7.jpg temp/1748362228_1472522_1360998888_8207b84bab9587a1613ded7faa563354.jpg temp/1748362228_1472522_1360998886_6c555bd7348100363b0f33ad03107bea.jpg temp/1748362228_1472522_1360998885_fe377f0eac3b8a0d093b69cfd46a0a47.jpg temp/1748362228_1472522_1360998884_1d10499d18c241b56bbd79318765557d.jpg temp/1748362228_1472522_1360998882_1430275e1b9a95f47888e61705a80942.jpg temp/1748362228_1472522_1360998854_8a7a79b47095a8d046c27de1165b0ad2.jpg temp/1748362228_1472522_1360998852_75d0eb07c510f060ec88628b6ed41dbc.jpg temp/1748362228_1472522_1360998850_554ff0e5bc6f6eca0d4d6886aa80b33f.jpg temp/1748362228_1472522_1360998847_c09193fdca2f5be45999f3747bf4d6e4.jpg temp/1748362228_1472522_1360998844_c8d406c52750a2f9cc67212524c082ff.jpg temp/1748362228_1472522_1360998842_13ceef1a91490a8c2c397b8b1cb28747.jpg temp/1748362228_1472522_1360998839_31c38c261c4a8e6ceedf041af0a51b89.jpg temp/1748362228_1472522_1360998837_e6ae1a5c73cebc5e78bebd96ecbe41a4.jpg temp/1748362228_1472522_1360998835_3392729ef05803c86f1a20254598ccce.jpg 136it [00:00, 382.78it/s] 197it [00:00, 455.72it/s] 200it [00:00, 456.21it/s]2025-05-27 18:11:23.575336: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2025-05-27 18:11:23.824334: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2025-05-27 18:11:24.058611: E tensorflow/stream_executor/cuda/cuda_dnn.cc:329] Could not create cudnn handle: CUDNN_STATUS_INTERNAL_ERROR 2025-05-27 18:11:24.069864: E tensorflow/stream_executor/cuda/cuda_dnn.cc:329] Could not create cudnn handle: CUDNN_STATUS_INTERNAL_ERROR temp/1748362228_1472522_1360998834_115ba74f4271ab47b0aeaa7de88bae29.jpg temp/1748362228_1472522_1360998833_66368df558d9af618ca08a76e0c41004.jpg temp/1748362228_1472522_1360998830_07d479c6f606e8c9ee2d7acb5148a0c2.jpg temp/1748362228_1472522_1360998818_6f323e87b13c32985fde91a6d967c80b.jpg temp/1748362228_1472522_1360998816_36e5469c88c173bc6970281f52406e80.jpg temp/1748362228_1472522_1360998814_9a6705d83bc9c94617c53c1d58e80dd0.jpg temp/1748362228_1472522_1360998813_9966c500c0a80caf49a8ffc7cab4db48.jpg temp/1748362228_1472522_1360998811_951e4e0202e10f879e52af5ccaab911c.jpg temp/1748362228_1472522_1360998809_dd0817ba58df530a5215a3fb33ce9f35.jpg temp/1748362228_1472522_1360998796_6d33521a1b4291c92b3f72a7801ca3b8.jpg temp/1748362228_1472522_1360998794_eff060558144625b53c411895f40d478.jpg temp/1748362228_1472522_1360998792_1dbdc794a3b2a0efd402e7095dc41897.jpg temp/1748362228_1472522_1360998791_a207e046b752d1817d88a4be5d86ebf1.jpg temp/1748362228_1472522_1360998789_c5ac8d954b59fd752732cb6ebc4fa8f2.jpg temp/1748362228_1472522_1360998788_978870ca8f33c3e3ea4c0ffd46b4a532.jpg temp/1748362228_1472522_1360998775_e42184ec0e7962cacc6a33506fea11b7.jpg temp/1748362228_1472522_1360998774_c9f6862b47ef82c216d090b87d4fb0f9.jpg temp/1748362228_1472522_1360998771_b374d3e30cab890280d3177bc6cce87b.jpg temp/1748362228_1472522_1360998769_523fd96bcffde0d93fa4d4dffbec1ba9.jpg temp/1748362228_1472522_1360998767_152251894417c7c74f0990c6ac237084.jpg temp/1748362228_1472522_1360998765_891aec6c8b6950a7d6b3e09003021e10.jpg temp/1748362228_1472522_1360998703_2ded1f831c4efa782554c1c45a9effaf.jpg temp/1748362228_1472522_1360998701_f135586054efca10f0f54433b330d7d5.jpg temp/1748362228_1472522_1360998699_0f1de6cba50a26dae3a669ccc432405a.jpg temp/1748362228_1472522_1360998696_ad6d31ee5bcd589fa7ee1e38600299f9.jpg temp/1748362228_1472522_1360998694_27699a3df86feeaf90f49c104fb3a421.jpg temp/1748362228_1472522_1360998692_5fce7e72453ef2819250d344f6c6a4ff.jpg temp/1748362228_1472522_1360998689_dae05cfe00671d2a7c14f3544c57e700.jpg temp/1748362228_1472522_1360998687_6f393ad54767fffbbbcf39b03f0aee33.jpg temp/1748362228_1472522_1360998686_8d2c63927ef9c78ca717fbbefe94c5d6.jpg temp/1748362228_1472522_1360998685_669bc195b148c19f140f3d5ece6aa191.jpg temp/1748362228_1472522_1360998684_1e58f6d3cfdf5866af4356163afc7f0e.jpg temp/1748362228_1472522_1360998682_89329ddccbc75d0ac3089553617b4924.jpg temp/1748362228_1472522_1360998656_7cffd5d7d3f0bef9c289b381fdb78768.jpg temp/1748362228_1472522_1360998654_a8636f46a11435fa0ec2fca10ac3c325.jpg temp/1748362228_1472522_1360998652_94207801434f9a2d551343052b331ce9.jpg temp/1748362228_1472522_1360998649_497738bc0555d8e42a36aaf40b19a1de.jpg temp/1748362228_1472522_1360998645_47d7e68805d40462bc8d4a800dbd50dc.jpg temp/1748362228_1472522_1360998642_7f17d0008a523a6822bb6ae2b081cfbc.jpg temp/1748362228_1472522_1360998630_75b2b26d80250e51faa2d7ef267bb951.jpg temp/1748362228_1472522_1360998628_77fdfb7ee1c49dffb6948edb6c8383ab.jpg temp/1748362228_1472522_1360998625_df14c4278bb0d2385f691237b8f45685.jpg temp/1748362228_1472522_1360998622_d0456c5ec52e9ca2601ecb8dccd399df.jpg temp/1748362228_1472522_1360998619_bdc53d9a96bb5f188e182d9955d34549.jpg temp/1748362228_1472522_1360998618_96ade74224f4071aa23618413ea15e71.jpg temp/1748362228_1472522_1360998616_bb602108575aefb6744910768e38cc22.jpg temp/1748362228_1472522_1360998614_00cbc20ba10b2c3f36c21bd52c395d69.jpg temp/1748362228_1472522_1360998611_0ac56f1177ba965e04205f246cf55dd3.jpg temp/1748362228_1472522_1360998609_4674646664bc14a124902a51b89b8327.jpg temp/1748362228_1472522_1360998607_061e379a755cf3ac2279b76d29936e0a.jpg temp/1748362228_1472522_1360998606_ce2a2276bc8c0e62e520e3a11b70c793.jpg temp/1748362228_1472522_1360998592_59ae8341dd2ab54985d1adbd128dc378.jpg temp/1748362228_1472522_1360998590_cabbefb2b7dd3c90a801d098e585478a.jpg temp/1748362228_1472522_1360998589_54070221d2022e0ff4c9bd6cead1f283.jpg temp/1748362228_1472522_1360998587_4c80590cc297100972ef21ce4d500e78.jpg temp/1748362228_1472522_1360998586_49eaabfcad78ce6cf8f2a22efc16a353.jpg temp/1748362228_1472522_1360998584_51d4164be548d1e264e4809ad1b4a805.jpg temp/1748362228_1472522_1360998571_e39c7f5ae0b30c39de320d54ceda015b.jpg temp/1748362228_1472522_1360998569_c5e271b912f2e9a2cf79e248e6aff655.jpg temp/1748362228_1472522_1360998565_ad147861637c420b65350d4563a32344.jpg temp/1748362228_1472522_1360998562_ea89872ea08bcb63fd31fa4cb3a7e26f.jpg temp/1748362228_1472522_1360998560_f236a921cf3995affa80a16a2a154d77.jpg temp/1748362228_1472522_1360998558_e2724885a83df85e8a780362eb2a54a1.jpg temp/1748362228_1472522_1360998545_547f1fbc8bb8f25fa46b7f6e9a44b8f2.jpg temp/1748362228_1472522_1360998543_704c4d20d4049fca43dc2bffacc0bf74.jpg temp/1748362228_1472522_1360998541_d8cbe45c57fb7f895918040a4eda6af2.jpg temp/1748362228_1472522_1360998540_091c6af64b277cdf6f22a23150a0fe22.jpg temp/1748362228_1472522_1360998539_a69f26cdb0f350aa41027629bf502029.jpg temp/1748362228_1472522_1360998537_91fa020f4ece3400ac623c9c94867d38.jpg temp/1748362228_1472522_1360998510_bb1219c1a41736d18fbc1a2e1368d3b6.jpg temp/1748362228_1472522_1360998508_0cea3cff4bd2351180a724f8508d3000.jpg temp/1748362228_1472522_1360998505_def7ba76cbc65cb8d74d4fe95c769051.jpg temp/1748362228_1472522_1360998502_5b475fd473195e3ec69aa0ee31b5c2c5.jpg temp/1748362228_1472522_1360998500_a7bc9cbaf772612e57d2dc24e5563235.jpg temp/1748362228_1472522_1360998498_fe9b31bd82390fdb3d7ca6bd72f68c57.jpg temp/1748362228_1472522_1360998495_a50bcfe8f90278a91d94050760d8a7f3.jpg temp/1748362228_1472522_1360998492_bc1f1d633e5bb830bc5a0632121e4a69.jpg temp/1748362228_1472522_1360998490_c03927d621de9ffa23a2176868f2d3d3.jpg temp/1748362228_1472522_1360998488_f5ced7db1fb3b14e56dd11bb713a786b.jpg temp/1748362228_1472522_1360998486_2e5d0310afd22cba712a49928f3bebc4.jpg temp/1748362228_1472522_1360998485_dbf8485e2d3193d7af205455741a8af4.jpg temp/1748362228_1472522_1360998459_ab5b86f6b48a9e0517e9198038c0cab7.jpg temp/1748362228_1472522_1360998456_a03fd339e8a2c832b2424581e2df2ec7.jpg temp/1748362228_1472522_1360998454_1bceda027554a98c62c103648d2098b4.jpg temp/1748362228_1472522_1360998451_708c100775d6be4b644ee173962762dd.jpg temp/1748362228_1472522_1360998449_1f0b4210eb7720b9fc9f20667a593265.jpg temp/1748362228_1472522_1360998448_a57378dee423f2649a13fb9c8c025a8a.jpg Found 200 images belonging to 1 classes. begin to do the prediction : ERROR in datou_step_exec, will save and exit ! Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above. [[{{node model/module/StatefulPartitionedCall/StatefulPartitionedCall/StatefulPartitionedCall/conv1/StatefulPartitionedCall/Conv2D}}]] [Op:__inference_predict_function_34490] Function call stack: predict_function File "/home/admin/workarea/git/Velours/python/mtr/datou/datou_lib.py", line 2329, in datou_exec output = datou_step_exec(sNext, args, cache, context, map_info, verbose, mtr_user_id) File "/home/admin/workarea/git/Velours/python/mtr/datou/datou_lib.py", line 2523, in datou_step_exec return lib_process.datou_step_tfhub2(param, json_param, args, cache, context, map_info, verbose) File "/home/admin/workarea/git/Velours/python/mtr/datou/lib_step_exec/lib_step_process.py", line 3147, in datou_step_tfhub2 classes, outputs, features = this_model.predict_image_paths(list_paths, keep_aspect_ratio=keep_aspect_ratio, File "/home/admin/workarea/git/Velours/python/mtr/tfhub2/evaluate.py", line 288, in predict_image_paths Y_pred, F_pred = self.model.predict(valid_generator, validation_steps) File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/training.py", line 88, in _method_wrapper return method(self, *args, **kwargs) File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/keras/engine/training.py", line 1268, in predict tmp_batch_outputs = predict_function(iterator) File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py", line 580, in __call__ result = self._call(*args, **kwds) File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/def_function.py", line 650, in _call return self._concrete_stateful_fn._filtered_call(canon_args, canon_kwds) # pylint: disable=protected-access File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py", line 1661, in _filtered_call return self._call_flat( File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py", line 1745, in _call_flat return self._build_call_outputs(self._inference_function.call( File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/function.py", line 593, in call outputs = execute.execute( File "/usr/local/lib/python3.8/dist-packages/tensorflow/python/eager/execute.py", line 59, in quick_execute tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name, [1360999259, 1360999257, 1360999242, 1360999239, 1360999237, 1360999236, 1360999235, 1360999234, 1360999209, 1360999207, 1360999205, 1360999203, 1360999202, 1360999201, 1360999198, 1360999196, 1360999194, 1360999193, 1360999192, 1360999191, 1360999184, 1360999182, 1360999180, 1360999178, 1360999175, 1360999174, 1360999161, 1360999159, 1360999158, 1360999156, 1360999155, 1360999154, 1360999136, 1360999133, 1360999131, 1360999130, 1360999129, 1360999127, 1360999115, 1360999112, 1360999111, 1360999110, 1360999109, 1360999108, 1360999096, 1360999094, 1360999093, 1360999092, 1360999091, 1360999090, 1360999076, 1360999073, 1360999071, 1360999070, 1360999069, 1360999068, 1360999056, 1360999054, 1360999052, 1360999051, 1360999050, 1360999048, 1360999021, 1360999019, 1360999016, 1360999013, 1360999012, 1360999010, 1360999007, 1360999005, 1360999004, 1360999003, 1360999002, 1360998999, 1360998987, 1360998985, 1360998982, 1360998980, 1360998979, 1360998977, 1360998964, 1360998961, 1360998959, 1360998957, 1360998956, 1360998954, 1360998941, 1360998938, 1360998935, 1360998931, 1360998929, 1360998926, 1360998922, 1360998921, 1360998920, 1360998919, 1360998918, 1360998917, 1360998890, 1360998888, 1360998886, 1360998885, 1360998884, 1360998882, 1360998854, 1360998852, 1360998850, 1360998847, 1360998844, 1360998842, 1360998839, 1360998837, 1360998835, 1360998834, 1360998833, 1360998830, 1360998818, 1360998816, 1360998814, 1360998813, 1360998811, 1360998809, 1360998796, 1360998794, 1360998792, 1360998791, 1360998789, 1360998788, 1360998775, 1360998774, 1360998771, 1360998769, 1360998767, 1360998765, 1360998703, 1360998701, 1360998699, 1360998696, 1360998694, 1360998692, 1360998689, 1360998687, 1360998686, 1360998685, 1360998684, 1360998682, 1360998656, 1360998654, 1360998652, 1360998649, 1360998645, 1360998642, 1360998630, 1360998628, 1360998625, 1360998622, 1360998619, 1360998618, 1360998616, 1360998614, 1360998611, 1360998609, 1360998607, 1360998606, 1360998592, 1360998590, 1360998589, 1360998587, 1360998586, 1360998584, 1360998571, 1360998569, 1360998565, 1360998562, 1360998560, 1360998558, 1360998545, 1360998543, 1360998541, 1360998540, 1360998539, 1360998537, 1360998510, 1360998508, 1360998505, 1360998502, 1360998500, 1360998498, 1360998495, 1360998492, 1360998490, 1360998488, 1360998486, 1360998485, 1360998459, 1360998456, 1360998454, 1360998451, 1360998449, 1360998448] begin to insert list_values into mtr_datou_result : length of list_values in save_final : 200 time used for this insertion : 0.10604166984558105 save_final ERROR in last step tfhub_classification2, Failed to get convolution algorithm. This is probably because cuDNN failed to initialize, so try looking to see if a warning log message was printed above. [[{{node model/module/StatefulPartitionedCall/StatefulPartitionedCall/StatefulPartitionedCall/conv1/StatefulPartitionedCall/Conv2D}}]] [Op:__inference_predict_function_34490] Function call stack: predict_function time spend for datou_step_exec : 18.976727724075317 time spend to save output : 0.1231231689453125 total time spend for step 0 : 19.09985089302063 need to delete datou_research and reload, so keep current state 1 caffe_path_current : About to save ! 2 After save, about to update current ! 13.81user 4.68system 1:00.78elapsed 30%CPU (0avgtext+0avgdata 2019936maxresident)k 182824inputs+299784outputs (480major+620956minor)pagefaults 0swaps