Hello. I'm trying to train the model using the COCO2017 dataset, as described in the README, but I get a fatal error every time I run the training script. I have Windows 11, Python 3.10, VS2019 16.11, CUDA 11.7, cuDNN 8.4.1, TensorRT 8.4.2.4 and PaddlePaddle 2.4.2 GPU version installed on my system. The error I get is as such:
Warning: import ppdet from source directory without installing, run 'python setup.py install' to install ppdet firstly
loading annotations into memory...
Done (t=10.89s)
creating index...
index created!
[05/20 16:48:40] ppdet.data.source.coco WARNING: Found an invalid bbox in annotations: im_id: 200365, area: 0.0 x1: 296.65, y1: 388.33, x2: 297.67999999999995, y2: 388.33.
[05/20 16:48:48] ppdet.data.source.coco WARNING: Found an invalid bbox in annotations: im_id: 550395, area: 0.0 x1: 9.98, y1: 188.56, x2: 15.52, y2: 188.56.
[05/20 16:48:50] ppdet.data.source.coco INFO: Load [117266 samples valid, 1021 samples invalid] in file C:\Users\mukka\Documents\Dev\coco2017\annotations\instances_train2017.json.
W0520 16:48:51.299536 9252 gpu_resources.cc:61] Please NOTE: device: 0, GPU Compute Capability: 8.6, Driver API Version: 11.7, Runtime API Version: 11.7
W0520 16:48:51.317999 9252 gpu_resources.cc:91] device: 0, cuDNN Version: 8.4.
[05/20 16:48:55] ppdet.utils.checkpoint INFO: Finish loading model weights: C:\Users\mukka/.cache/paddle/weights\ResNet18_vd_pretrained.pdparams
Fatal Python error: Aborted
Thread 0x00004524 (most recent call first):
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\fluid\dataloader\dataloader_iter.py", line 247 in _thread_loop
File "C:\Users\mukka\AppData\Local\Programs\Python\Python310\lib\threading.py", line 946 in run
File "C:\Users\mukka\AppData\Local\Programs\Python\Python310\lib\threading.py", line 1009 in _bootstrap_inner
File "C:\Users\mukka\AppData\Local\Programs\Python\Python310\lib\threading.py", line 966 in _bootstrap
Current thread 0x00002424 (most recent call first):
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\nn\functional\common.py", line 1886 in linear
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\nn\layer\common.py", line 175 in forward
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\fluid\dygraph\layers.py", line 1012 in call
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\heads\detr_head.py", line 50 in forward
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\fluid\dygraph\layers.py", line 1012 in call
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\transformers\rtdetr_transformerv3.py", line 608 in _get_decoder_input
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\transformers\rtdetr_transformerv3.py", line 514 in forward
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\fluid\dygraph\layers.py", line 1012 in call
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\architectures\rtdetrv3.py", line 95 in _forward
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\architectures\rtdetrv3.py", line 132 in get_loss
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\architectures\meta_arch.py", line 60 in forward
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\fluid\dygraph\layers.py", line 1012 in call
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\engine\trainer.py", line 634 in train
File "C:\Users\mukka\Desktop\RT-DETRv3\tools\train.py", line 161 in run
File "C:\Users\mukka\Desktop\RT-DETRv3\tools\train.py", line 208 in main
File "C:\Users\mukka\Desktop\RT-DETRv3\tools\train.py", line 212 in
Extension modules: numpy.core._multiarray_umath, numpy.core._multiarray_tests, numpy.linalg._umath_linalg, numpy.fft._pocketfft_internal, numpy.random._common, numpy.random.bit_generator, numpy.random._bounded_integers, numpy.random._mt19937, numpy.random.mtrand, numpy.random._philox, numpy.random._pcg64, numpy.random._sfc64, numpy.random._generator, cv2.cv2, google.protobuf.pyext._message, charset_normalizer.md, requests.packages.charset_normalizer.md, requests.packages.chardet.md, PIL._imaging, cython.cimports.libc.math, Cython.Utils, Cython.Plex.Actions, Cython.Plex.Transitions, Cython.Plex.Machines, Cython.Plex.DFA, Cython.Plex.Scanners, Cython.Compiler.Scanning, Cython.StringIOTree, Cython.Compiler.Code, yaml._yaml, pycocotools._mask, scipy._lib._ccallback_c, scipy.sparse._sparsetools, _csparsetools, scipy.sparse._csparsetools, scipy.linalg._fblas, scipy.linalg._flapack, scipy.linalg.cython_lapack, scipy.linalg._cythonized_array_utils, scipy.linalg._solve_toeplitz, scipy.linalg._decomp_lu_cython, scipy.linalg._matfuncs_sqrtm_triu, scipy.linalg._matfuncs_expm, scipy.linalg._linalg_pythran, scipy.linalg.cython_blas, scipy.linalg._decomp_update, scipy.sparse.linalg._dsolve._superlu, scipy.sparse.linalg._eigen.arpack._arpack, scipy.sparse.linalg._propack._spropack, scipy.sparse.linalg._propack._dpropack, scipy.sparse.linalg._propack._cpropack, scipy.sparse.linalg._propack._zpropack, scipy.sparse.csgraph._tools, scipy.sparse.csgraph._shortest_path, scipy.sparse.csgraph._traversal, scipy.sparse.csgraph._min_spanning_tree, scipy.sparse.csgraph._flow, scipy.sparse.csgraph._matching, scipy.sparse.csgraph._reordering, scipy.io.matlab._mio_utils, scipy.io.matlab._streams, scipy.io.matlab._mio5_utils, scipy.spatial._ckdtree, scipy._lib.messagestream, scipy.spatial._qhull, scipy.spatial._voronoi, scipy.spatial._distance_wrap, scipy.spatial._hausdorff, scipy.special._ufuncs_cxx, scipy.special._ufuncs, scipy.special._specfun, scipy.special._comb, scipy.special._ellip_harm_2, scipy.spatial.transform._rotation, scipy.optimize._group_columns, scipy.optimize._trlib._trlib, scipy.optimize._lbfgsb, _moduleTNC, scipy.optimize._moduleTNC, scipy.optimize._cobyla, scipy.optimize._slsqp, scipy.optimize._minpack, scipy.optimize._lsq.givens_elimination, scipy.optimize._zeros, scipy.optimize._cython_nnls, scipy._lib._uarray._uarray, scipy.linalg._decomp_interpolative, scipy.optimize._bglu_dense, scipy.optimize._lsap, scipy.optimize._direct, scipy.integrate._odepack, scipy.integrate._quadpack, scipy.integrate._vode, scipy.integrate._dop, scipy.integrate._lsoda, scipy.interpolate._fitpack, scipy.interpolate._dfitpack, scipy.interpolate._dierckx, scipy.interpolate._ppoly, scipy.interpolate._interpnd, scipy.interpolate._rbfinterp_pythran, scipy.interpolate._rgi_cython, scipy.interpolate._bspl, scipy.special.cython_special, scipy.stats._stats, scipy.stats._sobol, scipy.stats._qmc_cy, scipy.stats._biasedurn, scipy.stats._stats_pythran, scipy.stats._levy_stable.levyst, scipy.stats._ansari_swilk_statistics, scipy.stats._mvn, scipy.stats._rcont.rcont, scipy.ndimage._nd_image, scipy.ndimage._rank_filter_1d, _ni_label, scipy.ndimage._ni_label, skimage.measure._ccomp, skimage.segmentation._felzenszwalb_cy, scipy.cluster._vq, scipy.cluster._hierarchy, scipy.cluster._optimal_leaf_ordering, skimage.segmentation._slic, skimage.segmentation._quickshift_cy, skimage.morphology._misc_cy, _skeletonize_lee_cy, skimage.morphology._skeletonize_lee_cy, skimage.morphology._skeletonize_various_cy, skimage._shared.geometry, skimage.measure._pnpoly, skimage.morphology._convex_hull, skimage.morphology._grayreconstruct, skimage.morphology._extrema_cy, skimage.morphology._flood_fill_cy, skimage.morphology._max_tree, skimage.segmentation._watershed_cy, _lapjv, lap._lapjv, numba.core.typeconv._typeconv, numba._helperlib, numba._dynfunc, numba._dispatcher, numba.core.typing.builtins.itertools, numba.cpython.builtins.math, numba.core.runtime._nrt_python, numba.np.ufunc._internal, numba.experimental.jitclass._box, kiwisolver._cext, sklearn.__check_build._check_build, psutil._psutil_windows, pandas._libs.tslibs.ccalendar, pandas._libs.tslibs.np_datetime, pandas._libs.tslibs.dtypes, pandas._libs.tslibs.base, pandas._libs.tslibs.nattype, pandas._libs.tslibs.timezones, pandas._libs.tslibs.fields, pandas._libs.tslibs.timedeltas, pandas._libs.tslibs.tzconversion, pandas._libs.tslibs.timestamps, pandas._libs.properties, pandas._libs.tslibs.offsets, pandas._libs.tslibs.strptime, pandas._libs.tslibs.parsing, pandas._libs.tslibs.conversion, pandas._libs.tslibs.period, pandas._libs.tslibs.vectorized, pandas._libs.ops_dispatch, pandas._libs.missing, pandas._libs.hashtable, pandas._libs.algos, pandas._libs.interval, pandas._libs.lib, pandas._libs.ops, pandas._libs.hashing, pandas._libs.arrays, pandas._libs.tslib, pandas._libs.sparse, pandas._libs.internals, pandas._libs.indexing, pandas._libs.index, pandas._libs.writers, pandas._libs.join, pandas._libs.window.aggregations, pandas._libs.window.indexers, pandas._libs.reshape, pandas._libs.groupby, pandas._libs.json, pandas._libs.parsers, pandas._libs.testing, sklearn.utils._isfinite, sklearn.utils.sparsefuncs_fast, sklearn.utils.murmurhash, sklearn.utils._openmp_helpers, PIL._imagingft, shapely.lib, shapely._geos, shapely._geometry_helpers (total: 198)
Hello. I'm trying to train the model using the COCO2017 dataset, as described in the README, but I get a fatal error every time I run the training script. I have Windows 11, Python 3.10, VS2019 16.11, CUDA 11.7, cuDNN 8.4.1, TensorRT 8.4.2.4 and PaddlePaddle 2.4.2 GPU version installed on my system. The error I get is as such:
Warning: import ppdet from source directory without installing, run 'python setup.py install' to install ppdet firstly
loading annotations into memory...
Done (t=10.89s)
creating index...
index created!
[05/20 16:48:40] ppdet.data.source.coco WARNING: Found an invalid bbox in annotations: im_id: 200365, area: 0.0 x1: 296.65, y1: 388.33, x2: 297.67999999999995, y2: 388.33.
[05/20 16:48:48] ppdet.data.source.coco WARNING: Found an invalid bbox in annotations: im_id: 550395, area: 0.0 x1: 9.98, y1: 188.56, x2: 15.52, y2: 188.56.
[05/20 16:48:50] ppdet.data.source.coco INFO: Load [117266 samples valid, 1021 samples invalid] in file C:\Users\mukka\Documents\Dev\coco2017\annotations\instances_train2017.json.
W0520 16:48:51.299536 9252 gpu_resources.cc:61] Please NOTE: device: 0, GPU Compute Capability: 8.6, Driver API Version: 11.7, Runtime API Version: 11.7
W0520 16:48:51.317999 9252 gpu_resources.cc:91] device: 0, cuDNN Version: 8.4.
[05/20 16:48:55] ppdet.utils.checkpoint INFO: Finish loading model weights: C:\Users\mukka/.cache/paddle/weights\ResNet18_vd_pretrained.pdparams
Fatal Python error: Aborted
Thread 0x00004524 (most recent call first):
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\fluid\dataloader\dataloader_iter.py", line 247 in _thread_loop
File "C:\Users\mukka\AppData\Local\Programs\Python\Python310\lib\threading.py", line 946 in run
File "C:\Users\mukka\AppData\Local\Programs\Python\Python310\lib\threading.py", line 1009 in _bootstrap_inner
File "C:\Users\mukka\AppData\Local\Programs\Python\Python310\lib\threading.py", line 966 in _bootstrap
Current thread 0x00002424 (most recent call first):
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\nn\functional\common.py", line 1886 in linear
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\nn\layer\common.py", line 175 in forward
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\fluid\dygraph\layers.py", line 1012 in call
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\heads\detr_head.py", line 50 in forward
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\fluid\dygraph\layers.py", line 1012 in call
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\transformers\rtdetr_transformerv3.py", line 608 in _get_decoder_input
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\transformers\rtdetr_transformerv3.py", line 514 in forward
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\fluid\dygraph\layers.py", line 1012 in call
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\architectures\rtdetrv3.py", line 95 in _forward
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\architectures\rtdetrv3.py", line 132 in get_loss
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\modeling\architectures\meta_arch.py", line 60 in forward
File "C:\Users\mukka\Desktop\detr\lib\site-packages\paddle\fluid\dygraph\layers.py", line 1012 in call
File "C:\Users\mukka\Desktop\RT-DETRv3\ppdet\engine\trainer.py", line 634 in train
File "C:\Users\mukka\Desktop\RT-DETRv3\tools\train.py", line 161 in run
File "C:\Users\mukka\Desktop\RT-DETRv3\tools\train.py", line 208 in main
File "C:\Users\mukka\Desktop\RT-DETRv3\tools\train.py", line 212 in
Extension modules: numpy.core._multiarray_umath, numpy.core._multiarray_tests, numpy.linalg._umath_linalg, numpy.fft._pocketfft_internal, numpy.random._common, numpy.random.bit_generator, numpy.random._bounded_integers, numpy.random._mt19937, numpy.random.mtrand, numpy.random._philox, numpy.random._pcg64, numpy.random._sfc64, numpy.random._generator, cv2.cv2, google.protobuf.pyext._message, charset_normalizer.md, requests.packages.charset_normalizer.md, requests.packages.chardet.md, PIL._imaging, cython.cimports.libc.math, Cython.Utils, Cython.Plex.Actions, Cython.Plex.Transitions, Cython.Plex.Machines, Cython.Plex.DFA, Cython.Plex.Scanners, Cython.Compiler.Scanning, Cython.StringIOTree, Cython.Compiler.Code, yaml._yaml, pycocotools._mask, scipy._lib._ccallback_c, scipy.sparse._sparsetools, _csparsetools, scipy.sparse._csparsetools, scipy.linalg._fblas, scipy.linalg._flapack, scipy.linalg.cython_lapack, scipy.linalg._cythonized_array_utils, scipy.linalg._solve_toeplitz, scipy.linalg._decomp_lu_cython, scipy.linalg._matfuncs_sqrtm_triu, scipy.linalg._matfuncs_expm, scipy.linalg._linalg_pythran, scipy.linalg.cython_blas, scipy.linalg._decomp_update, scipy.sparse.linalg._dsolve._superlu, scipy.sparse.linalg._eigen.arpack._arpack, scipy.sparse.linalg._propack._spropack, scipy.sparse.linalg._propack._dpropack, scipy.sparse.linalg._propack._cpropack, scipy.sparse.linalg._propack._zpropack, scipy.sparse.csgraph._tools, scipy.sparse.csgraph._shortest_path, scipy.sparse.csgraph._traversal, scipy.sparse.csgraph._min_spanning_tree, scipy.sparse.csgraph._flow, scipy.sparse.csgraph._matching, scipy.sparse.csgraph._reordering, scipy.io.matlab._mio_utils, scipy.io.matlab._streams, scipy.io.matlab._mio5_utils, scipy.spatial._ckdtree, scipy._lib.messagestream, scipy.spatial._qhull, scipy.spatial._voronoi, scipy.spatial._distance_wrap, scipy.spatial._hausdorff, scipy.special._ufuncs_cxx, scipy.special._ufuncs, scipy.special._specfun, scipy.special._comb, scipy.special._ellip_harm_2, scipy.spatial.transform._rotation, scipy.optimize._group_columns, scipy.optimize._trlib._trlib, scipy.optimize._lbfgsb, _moduleTNC, scipy.optimize._moduleTNC, scipy.optimize._cobyla, scipy.optimize._slsqp, scipy.optimize._minpack, scipy.optimize._lsq.givens_elimination, scipy.optimize._zeros, scipy.optimize._cython_nnls, scipy._lib._uarray._uarray, scipy.linalg._decomp_interpolative, scipy.optimize._bglu_dense, scipy.optimize._lsap, scipy.optimize._direct, scipy.integrate._odepack, scipy.integrate._quadpack, scipy.integrate._vode, scipy.integrate._dop, scipy.integrate._lsoda, scipy.interpolate._fitpack, scipy.interpolate._dfitpack, scipy.interpolate._dierckx, scipy.interpolate._ppoly, scipy.interpolate._interpnd, scipy.interpolate._rbfinterp_pythran, scipy.interpolate._rgi_cython, scipy.interpolate._bspl, scipy.special.cython_special, scipy.stats._stats, scipy.stats._sobol, scipy.stats._qmc_cy, scipy.stats._biasedurn, scipy.stats._stats_pythran, scipy.stats._levy_stable.levyst, scipy.stats._ansari_swilk_statistics, scipy.stats._mvn, scipy.stats._rcont.rcont, scipy.ndimage._nd_image, scipy.ndimage._rank_filter_1d, _ni_label, scipy.ndimage._ni_label, skimage.measure._ccomp, skimage.segmentation._felzenszwalb_cy, scipy.cluster._vq, scipy.cluster._hierarchy, scipy.cluster._optimal_leaf_ordering, skimage.segmentation._slic, skimage.segmentation._quickshift_cy, skimage.morphology._misc_cy, _skeletonize_lee_cy, skimage.morphology._skeletonize_lee_cy, skimage.morphology._skeletonize_various_cy, skimage._shared.geometry, skimage.measure._pnpoly, skimage.morphology._convex_hull, skimage.morphology._grayreconstruct, skimage.morphology._extrema_cy, skimage.morphology._flood_fill_cy, skimage.morphology._max_tree, skimage.segmentation._watershed_cy, _lapjv, lap._lapjv, numba.core.typeconv._typeconv, numba._helperlib, numba._dynfunc, numba._dispatcher, numba.core.typing.builtins.itertools, numba.cpython.builtins.math, numba.core.runtime._nrt_python, numba.np.ufunc._internal, numba.experimental.jitclass._box, kiwisolver._cext, sklearn.__check_build._check_build, psutil._psutil_windows, pandas._libs.tslibs.ccalendar, pandas._libs.tslibs.np_datetime, pandas._libs.tslibs.dtypes, pandas._libs.tslibs.base, pandas._libs.tslibs.nattype, pandas._libs.tslibs.timezones, pandas._libs.tslibs.fields, pandas._libs.tslibs.timedeltas, pandas._libs.tslibs.tzconversion, pandas._libs.tslibs.timestamps, pandas._libs.properties, pandas._libs.tslibs.offsets, pandas._libs.tslibs.strptime, pandas._libs.tslibs.parsing, pandas._libs.tslibs.conversion, pandas._libs.tslibs.period, pandas._libs.tslibs.vectorized, pandas._libs.ops_dispatch, pandas._libs.missing, pandas._libs.hashtable, pandas._libs.algos, pandas._libs.interval, pandas._libs.lib, pandas._libs.ops, pandas._libs.hashing, pandas._libs.arrays, pandas._libs.tslib, pandas._libs.sparse, pandas._libs.internals, pandas._libs.indexing, pandas._libs.index, pandas._libs.writers, pandas._libs.join, pandas._libs.window.aggregations, pandas._libs.window.indexers, pandas._libs.reshape, pandas._libs.groupby, pandas._libs.json, pandas._libs.parsers, pandas._libs.testing, sklearn.utils._isfinite, sklearn.utils.sparsefuncs_fast, sklearn.utils.murmurhash, sklearn.utils._openmp_helpers, PIL._imagingft, shapely.lib, shapely._geos, shapely._geometry_helpers (total: 198)