diff --git a/BUILD.bazel b/BUILD.bazel index 685ec389c9..fbd2d31494 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -48,8 +48,6 @@ cc_library( "@aws-sdk-cpp//:aws-sdk-cpp", "@aws-sdk-cpp//:aws-sdk-cpp_cmake", "@azure//:storage", - "@cpprest//:sdk", - "@boost//:boost", "@google_cloud_cpp//:storage",], "//:disable_cloud": [] }) diff --git a/Dockerfile.redhat b/Dockerfile.redhat index 0e8e39c795..8e6e289c68 100644 --- a/Dockerfile.redhat +++ b/Dockerfile.redhat @@ -48,21 +48,53 @@ RUN echo -e "max_parallel_downloads=8\nretries=50" >> /etc/dnf/dnf.conf && \ wget && \ dnf clean all -####### Azure SDK needs new boost: -WORKDIR /boost -# hadolint ignore=DL3003 -RUN wget -nv https://sourceforge.net/projects/boost/files/boost/1.68.0/boost_1_68_0.tar.gz && \ - if [ "$VERBOSE_LOGS" == "ON" ] ; then export BVERBOSE="-d+2" ; else export BVERBOSE="" ; fi && \ - tar xf boost_1_68_0.tar.gz && cd boost_1_68_0 && ./bootstrap.sh && \ - ./b2 -j ${JOBS} cxxstd=17 link=static variant=release \ - cxxflags="-fPIC $LTO_CXX_FLAGS" \ - cflags="-fPIC $LTO_CXX_FLAGS" \ - linkflags="$LTO_LD_FLAGS" \ - --with-chrono --with-date_time --with-filesystem \ - --with-program_options --with-system \ - --with-random --with-thread --with-atomic --with-regex \ - --with-log --with-locale "$BVERBOSE" \ - install +####### Azure SDK (modern azure-sdk-for-cpp: azure-core, azure-storage-blobs, azure-storage-files-shares) +ARG SDL_OPS="-Wl,-z,relro,-z,now,-z,noexecstack -Wall -Wextra -Wimplicit-fallthrough -fPIE -pie -fstack-protector-strong -fexceptions -fasynchronous-unwind-tables -fcf-protection -fpic -O2 -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -fno-strict-overflow -Wall -Wno-unknown-pragmas -Wno-error=sign-compare -fno-delete-null-pointer-checks -fwrapv -fstack-clash-protection -Wformat -Wformat-security -Werror=format-security -s -D_GLIBCXX_USE_CXX11_ABI=1 -Wuninitialized" +WORKDIR /azure-sdk-for-cpp +RUN git clone --depth 1 --branch azure-storage-blobs_12.18.0 https://github.com/Azure/azure-sdk-for-cpp.git . && \ + echo "# no-op: use system libcurl and openssl" > /azure-sdk-toolchain.cmake && \ + cmake -S sdk/core/azure-core -B build/azure-core \ + -DCMAKE_TOOLCHAIN_FILE=/azure-sdk-toolchain.cmake \ + -DCMAKE_CXX_FLAGS="${SDL_OPS} ${LTO_CXX_FLAGS}" \ + -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTING=OFF \ + -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON \ + -DBUILD_TRANSPORT_CURL=ON && \ + cmake --build build/azure-core --parallel ${JOBS} && \ + cmake --install build/azure-core --prefix /azure-sdk-install && \ + cmake -S sdk/storage/azure-storage-common -B build/azure-storage-common \ + -DCMAKE_TOOLCHAIN_FILE=/azure-sdk-toolchain.cmake \ + -DCMAKE_CXX_FLAGS="${SDL_OPS} ${LTO_CXX_FLAGS}" \ + -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_PREFIX_PATH=/azure-sdk-install && \ + cmake --build build/azure-storage-common --parallel ${JOBS} && \ + cmake --install build/azure-storage-common --prefix /azure-sdk-install && \ + cmake -S sdk/storage/azure-storage-blobs -B build/azure-storage-blobs \ + -DCMAKE_TOOLCHAIN_FILE=/azure-sdk-toolchain.cmake \ + -DCMAKE_CXX_FLAGS="${SDL_OPS} ${LTO_CXX_FLAGS}" \ + -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_PREFIX_PATH=/azure-sdk-install && \ + cmake --build build/azure-storage-blobs --parallel ${JOBS} && \ + cmake --install build/azure-storage-blobs --prefix /azure-sdk-install && \ + cmake -S sdk/storage/azure-storage-files-shares -B build/azure-storage-files-shares \ + -DCMAKE_TOOLCHAIN_FILE=/azure-sdk-toolchain.cmake \ + -DCMAKE_CXX_FLAGS="${SDL_OPS} ${LTO_CXX_FLAGS}" \ + -DCMAKE_SHARED_LINKER_FLAGS="${LTO_LD_FLAGS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_PREFIX_PATH=/azure-sdk-install && \ + cmake --build build/azure-storage-files-shares --parallel ${JOBS} && \ + cmake --install build/azure-storage-files-shares --prefix /azure-sdk-install +####### End of Azure SDK COPY third_party/pugixml /ovms/third_party/pugixml @@ -77,23 +109,6 @@ RUN git clone -b v1.13 https://github.com/zeux/pugixml && \ make all -j ${JOBS} && \ cp -P libpugixml.so* /usr/lib64/ -####### Azure SDK -WORKDIR /azure -COPY third_party/azure /ovms/third_party/azure -COPY third_party/cpprest /ovms/third_party/cpprest -RUN git clone --recurse-submodules --depth 1 --branch v2.10.16 https://github.com/Microsoft/cpprestsdk.git && \ - git clone --depth 1 --branch v7.5.0 https://github.com/Azure/azure-storage-cpp.git && \ - patch -d /azure/cpprestsdk/ -p1 < /ovms/third_party/cpprest/rest_sdk_v2.10.16.patch && \ - patch -d /azure/azure-storage-cpp/ -p1 0|#ifdef PTHREAD_STACK_MIN|g' boost/thread/pthread/thread_data.hpp && \ -# fix for compiler >=9.5 https://github.com/boostorg/thread/pull/297/files -./b2 -j ${JOBS} cxxstd=17 link=static cxxflags='-fPIC' cflags='-fPIC' \ ---with-chrono --with-date_time --with-filesystem --with-program_options --with-system \ ---with-random --with-thread --with-atomic --with-regex \ ---with-log --with-locale \ -install - -COPY third_party/azure /ovms/third_party/azure -COPY third_party/cpprest /ovms/third_party/cpprest - -####### Azure SDK -WORKDIR /azure -RUN apt-get update && apt-get install --no-install-recommends -y uuid uuid-dev && rm -rf /var/lib/apt/lists/* -RUN git clone --recurse-submodules --depth 1 --branch v2.10.16 https://github.com/Microsoft/cpprestsdk.git && \ - git clone --depth 1 --branch v7.5.0 https://github.com/Azure/azure-storage-cpp.git && \ - patch -d /azure/cpprestsdk/ -p1 < /ovms/third_party/cpprest/rest_sdk_v2.10.16.patch && \ - patch -d /azure/azure-storage-cpp/ -p1 /azure-sdk-toolchain.cmake && \ + cmake -S sdk/core/azure-core -B build/azure-core \ + -DCMAKE_TOOLCHAIN_FILE=/azure-sdk-toolchain.cmake \ + -DCMAKE_CXX_FLAGS="${SDL_OPS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTING=OFF \ + -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON \ + -DBUILD_TRANSPORT_CURL=ON && \ + cmake --build build/azure-core --parallel ${JOBS} && \ + cmake --install build/azure-core --prefix /azure-sdk-install && \ + cmake -S sdk/storage/azure-storage-common -B build/azure-storage-common \ + -DCMAKE_TOOLCHAIN_FILE=/azure-sdk-toolchain.cmake \ + -DCMAKE_CXX_FLAGS="${SDL_OPS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_PREFIX_PATH=/azure-sdk-install && \ + cmake --build build/azure-storage-common --parallel ${JOBS} && \ + cmake --install build/azure-storage-common --prefix /azure-sdk-install && \ + cmake -S sdk/storage/azure-storage-blobs -B build/azure-storage-blobs \ + -DCMAKE_TOOLCHAIN_FILE=/azure-sdk-toolchain.cmake \ + -DCMAKE_CXX_FLAGS="${SDL_OPS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_PREFIX_PATH=/azure-sdk-install && \ + cmake --build build/azure-storage-blobs --parallel ${JOBS} && \ + cmake --install build/azure-storage-blobs --prefix /azure-sdk-install && \ + cmake -S sdk/storage/azure-storage-files-shares -B build/azure-storage-files-shares \ + -DCMAKE_TOOLCHAIN_FILE=/azure-sdk-toolchain.cmake \ + -DCMAKE_CXX_FLAGS="${SDL_OPS}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_TESTING=OFF \ + -DCMAKE_PREFIX_PATH=/azure-sdk-install && \ + cmake --build build/azure-storage-files-shares --parallel ${JOBS} && \ + cmake --install build/azure-storage-files-shares --prefix /azure-sdk-install ####### End of Azure SDK ####### Build OpenCV @@ -320,9 +333,6 @@ ARG CHECK_COVERAGE=0 COPY run_unit_tests.sh prepare_llm_models.sh install_va.sh demos/common/export_models/export_model.py /ovms/ ARG FUZZER_BUILD=0 -# Custom Nodes -# hadolint ignore=DL3059 -RUN if [ "$FUZZER_BUILD" == "0" ]; then bazel build --jobs=$JOBS ${debug_bazel_flags} //src:release_custom_nodes; fi; # OVMS ARG OPTIMIZE_BUILDING_TESTS=0 diff --git a/Makefile b/Makefile index 2fcc40b417..aebb6bbdf8 100644 --- a/Makefile +++ b/Makefile @@ -211,8 +211,6 @@ PYTHON_CLIENT_TEST_CONTAINER_NAME ?= python-client-test$(shell date +%Y-%m-%d-%H TEST_PATH ?= tests/functional/ -BUILD_CUSTOM_NODES ?= false - VERBOSE_LOGS ?= OFF BUILD_ARGS = --build-arg http_proxy=$(HTTP_PROXY)\ @@ -361,10 +359,6 @@ ifeq ($(NO_DOCKER_CACHE),true) endif endif -ifeq ($(BUILD_CUSTOM_NODES),true) - @echo "Building custom nodes" - @cd src/custom_nodes && make NO_DOCKER_CACHE=$(NO_DOCKER_CACHE) BASE_OS=$(OS) BASE_IMAGE=$(BASE_IMAGE) -endif @echo "Building docker image $(BASE_OS)" # Provide metadata information into image if defined @mkdir -p .workspace diff --git a/MakefileCapi b/MakefileCapi index b23230e1bc..6e001d8c81 100644 --- a/MakefileCapi +++ b/MakefileCapi @@ -16,11 +16,12 @@ cpp: bazel build --jobs=${JOBS} ${CAPI_FLAGS} //src:ovms_shared - g++ src/main_capi.cpp -I/ovms/src/ -L/ovms/bazel-bin/src/ -lovms_shared -fPIC --std=c++17 -o /ovms/bazel-bin/src/capi_cpp_example - LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/bazel-bin/src/ /ovms/bazel-bin/src/capi_cpp_example + g++ src/main_capi.cpp -I/ovms/src/ -L/ovms/bazel-bin/src/ -L/azure-sdk-install/lib \ + -Wl,-rpath-link,/azure-sdk-install/lib -lovms_shared -fPIC --std=c++17 -o /ovms/bazel-bin/src/capi_cpp_example + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/bazel-bin/src/:/azure-sdk-install/lib /ovms/bazel-bin/src/capi_cpp_example c: bazel build --jobs=${JOBS} ${CAPI_FLAGS} //src:ovms_shared gcc -c src/main_capi.c -o /ovms/bazel-bin/src/main_capi.o -std=c99 - gcc -o /ovms/bazel-bin/src/capi_c_example /ovms/bazel-bin/src/main_capi.o -lovms_shared -L/ovms/bazel-bin/src/ - LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/bazel-bin/src/ /ovms/bazel-bin/src/capi_c_example + gcc -o /ovms/bazel-bin/src/capi_c_example /ovms/bazel-bin/src/main_capi.o -lovms_shared -L/ovms/bazel-bin/src/ -L/azure-sdk-install/lib -Wl,-rpath-link,/azure-sdk-install/lib + LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/ovms/bazel-bin/src/:/azure-sdk-install/lib /ovms/bazel-bin/src/capi_c_example diff --git a/WORKSPACE b/WORKSPACE index 558c4fcdf0..d16f2c09ac 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -483,26 +483,11 @@ drogon_cpp() load("@ovms//third_party/espeak_ng:espeak_ng.bzl", "espeak_ng") espeak_ng() -# Azure Storage SDK +# Azure SDK for C++ (azure-core, azure-storage-blobs, azure-storage-files-shares) new_local_repository( name = "azure", build_file = "@//third_party/azure:BUILD", - path = "/azure/azure-storage-cpp", -) - -# Azure Storage SDK dependency - cpprest -new_local_repository( - name = "cpprest", - build_file = "@//third_party/cpprest:BUILD", - path = "/azure/cpprestsdk", -) - -# Boost (needed for Azure Storage SDK) - -new_local_repository( - name = "boost", - path = "/usr/local/lib/", - build_file = "@//third_party/boost:BUILD" + path = "/azure-sdk-install", ) # Google Cloud SDK diff --git a/create_package.sh b/create_package.sh index 93d6bf78a5..57acc05d9d 100755 --- a/create_package.sh +++ b/create_package.sh @@ -21,12 +21,19 @@ set -e env mkdir -vp /ovms_release/bin mkdir -vp /ovms_release/lib -mkdir -vp /ovms_release/lib/custom_nodes # Do not link this tokenizer lib as it has old protobuf sentencepiece symbols the conflict with new protobuf from ovsm if [ "$ov_use_binary" == "0" ] ; then cp -v /openvino_tokenizers/build/src/libopenvino_tokenizers.so /ovms_release/lib/ ; fi -find /ovms/bazel-out/k8-*/bin -iname '*.so*' ! -type d ! -name "libgtest.so" ! -name "*params" ! -name "*.hana.*" ! -name "py_generate_pipeline.cpython*" ! -name "lib_node_*" ! -path "*test_python_binding*" ! -name "*libpython*" -exec cp -v {} /ovms_release/lib/ \; +find /ovms/bazel-out/k8-*/bin -iname '*.so*' ! -type d ! -name "libgtest.so" ! -name "*params" ! -name "*.hana.*" ! -name "py_generate_pipeline.cpython*" ! -name "lib_node_*" ! -name "libazure-*" ! -name "pyovms.so" ! -path "*/_solib_k8/*" ! -path "*test_python_binding*" ! -name "*libpython*" -exec cp -vP {} /ovms_release/lib/ \; +# Copy pyovms.so directly as a file (not symlink) to avoid broken Bazel cache paths +find /ovms/bazel-out/k8-*/bin/src/python/binding -name 'pyovms.so' -type f -exec cp -v {} /ovms_release/lib/ \; + +# Copy Azure SDK libs directly from the CMake install prefix so that the +# unversioned .so files are local relative symlinks (not absolute Bazel cache +# paths), avoiding duplicate regular-file copies of the same content. +find /azure-sdk-install/lib -maxdepth 1 -name 'libazure-*.so*' -exec cp -vP {} /ovms_release/lib/ \; + # Bundle espeak-ng data files when espeak was enabled in the Bazel build. # rules_foreign_cc places the cmake install tree under copy_/espeak-ng/ @@ -52,10 +59,7 @@ if [ -n "$ESPEAK_REAL" ]; then ln -s libespeak-ng.so.1 libespeak-ng.so cd - >/dev/null fi -if [ "$FUZZER_BUILD" == "0" ]; then mv /ovms_release/lib/libcustom_node* /ovms_release/lib/custom_nodes/; fi; cd /ovms_release/lib/ ; rm -f libcurl.so* -cd /ovms_release/lib/ ; rm -f libazurestorage.so.* ; ln -s libazurestorage.so libazurestorage.so.7 ;ln -s libazurestorage.so libazurestorage.so.7.5 -cd /ovms_release/lib/ ; rm -f libcpprest.so.2.10 ; ln -s libcpprest.so libcpprest.so.2.10 # Remove GPU plugin for CPU images? # Remove OpenCL for CPU images? diff --git a/src/BUILD b/src/BUILD index dfa47dc911..a50b90a904 100644 --- a/src/BUILD +++ b/src/BUILD @@ -41,18 +41,6 @@ config_setting( visibility = ["//visibility:public"], ) -cc_library( - name = "custom_nodes_common_buffersqueue", - linkstatic = 1, - hdrs = ["custom_nodes/common/buffersqueue.hpp"], - srcs = [ - "custom_nodes/common/buffersqueue.hpp", - "custom_nodes/common/buffersqueue.cpp", - ], - deps = ["libovms_queue"], - copts = COMMON_STATIC_LIBS_COPTS, -) - ovms_cc_library( name = "executor_base", hdrs = ["executor_base.hpp"], @@ -73,7 +61,6 @@ cc_shared_library( "@fmtlib//:__subpackages__", "@com_github_jarro2783_cxxopts//:__subpackages__", "@aws-sdk-cpp//:__subpackages__", - "@boost//:__subpackages__", "@com_github_googleapis_google_cloud_cpp//:__subpackages__", "//third_party:curl//:__subpackages__", "@com_github_google_crc32c//:__subpackages__", @@ -108,7 +95,6 @@ cc_shared_library( "@flatbuffers//:__subpackages__", "@ruy//:__subpackages__", "@azure//:__subpackages__", - "@cpprest//:__subpackages__", "//third_party:opencv//:__subpackages__", ] + select({ "//:not_disable_mediapipe": [ @@ -1060,6 +1046,16 @@ ovms_cc_library( ], visibility = ["//visibility:public",], ) +ovms_cc_library( + name = "opencv_dep", + srcs = [ + "deps/opencv.hpp", + ], + deps = [ + "//third_party:opencv", + ], + visibility = ["//visibility:public",], +) ovms_cc_library( name = "tensor_conversion_common", hdrs = ["tensor_conversion_common.hpp"], @@ -1157,115 +1153,6 @@ cc_binary( linkshared = 1, ) -cc_binary( - name = "libcustom_node_east_ocr.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/opencv_utils.hpp", - "custom_nodes/east_ocr/east_ocr.cpp", - "custom_nodes/east_ocr/nms.hpp", - "custom_node_interface.h", - ], - deps = [ - "opencv_dep", - ], - linkshared = 1, - copts = COMMON_STATIC_LIBS_COPTS, -) - -cc_binary( - name = "libcustom_node_face_blur.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/opencv_utils.hpp", - "custom_nodes/face_blur/face_blur.cpp", - "custom_node_interface.h", - ], - deps = [ - "opencv_dep", - ], - linkshared = 1, - copts =COMMON_STATIC_LIBS_COPTS, -) - -cc_binary( - name = "libcustom_node_add_one.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/custom_node_library_internal_manager.hpp", - "custom_nodes/common/custom_node_library_internal_manager.cpp", - "custom_nodes/add_one/add_one.cpp", - "custom_node_interface.h", - "custom_nodes/add_one/add_one_internal_manager.hpp" - ], - deps = [ - "custom_nodes_common_buffersqueue", - "libovms_queue", - ], - linkshared = 1, - copts = COMMON_STATIC_LIBS_COPTS, -) - -ovms_cc_library( - name = "opencv_dep", - srcs = [ - "deps/opencv.hpp", - ], - deps = [ - "//third_party:opencv", - ], - visibility = ["//visibility:public",], -) - -cc_binary( - name = "libcustom_node_model_zoo_intel_object_detection.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/opencv_utils.hpp", - "custom_nodes/common/custom_node_library_internal_manager.hpp", - "custom_nodes/common/custom_node_library_internal_manager.cpp", - "custom_nodes/model_zoo_intel_object_detection/model_zoo_intel_object_detection.cpp", - "custom_node_interface.h", - ], - deps = [ - "custom_nodes_common_buffersqueue", - "opencv_dep", - "libovms_queue", - ], - linkshared = 1, - copts = COMMON_STATIC_LIBS_COPTS, -) - -cc_binary( - name = "libcustom_node_image_transformation.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/opencv_utils.hpp", - "custom_nodes/image_transformation/image_transformation.cpp", - "custom_node_interface.h", - ], - deps = [ - "opencv_dep", - ], - linkshared = 1, - copts = COMMON_STATIC_LIBS_COPTS, -) - -cc_binary( - name = "libcustom_node_horizontal_ocr.so", - srcs = [ - "custom_nodes/common/utils.hpp", - "custom_nodes/common/opencv_utils.hpp", - "custom_nodes/horizontal_ocr/horizontal_ocr.cpp", - "custom_node_interface.h", - ], - deps = [ - "opencv_dep", - ], - linkshared = 1, - copts = COMMON_STATIC_LIBS_COPTS, -) - # POC which can start OVMS with C-API built by bazel # Standalone example for building outside of bazel (with bare g++ is inside MakefileCapi) cc_binary( @@ -2054,7 +1941,13 @@ cc_binary( "-luuid", "-lstdc++fs", "-lcrypto", - ], + ] + select({ + "//conditions:default": [ + "-Wl,-rpath,$$ORIGIN/capi_benchmark.runfiles/ovms/external/azure/lib", + "-Wl,-rpath,$$ORIGIN/capi_benchmark.runfiles/azure/lib", + ], + "//src:windows": [], + }), copts = [ ], deps = [ @@ -2082,19 +1975,19 @@ cc_binary( ] + select({ "//conditions:default": [], "//src:windows" : ["main_windows.hpp", "main_windows.cpp",],}), - linkopts = LINKOPTS_ADJUSTED, + linkopts = LINKOPTS_ADJUSTED + select({ + "//conditions:default": [ + "-Wl,-rpath,$$ORIGIN/ovms.runfiles/ovms/external/azure/lib", + "-Wl,-rpath,$$ORIGIN/ovms.runfiles/azure/lib", + ], + "//src:windows": [], + }), copts = COMMON_STATIC_LIBS_COPTS, deps = [ "//src:ovms_lib", "//src/filesystem:libovmsfilesystemfactory",] + select({ - "//conditions:default": [ - "//src:libcustom_node_east_ocr.so", - "//src:libcustom_node_face_blur.so", - "//src:libcustom_node_horizontal_ocr.so", - "//src:libcustom_node_model_zoo_intel_object_detection.so", - "//src:libcustom_node_image_transformation.so", - ], + "//conditions:default": [], "//src:windows" : ["@winreg//:winreg"], }) + select({ @@ -2120,8 +2013,6 @@ cc_test( "test/c_api_tests.cpp", "test/capi_predict_validation_test.cpp", #"test/custom_loader_test.cpp", # TODO remove? - "test/custom_node_buffersqueue_test.cpp", - "test/custom_node_output_allocator_test.cpp", "test/demultiplexer_node_test.cpp", "test/deserialization_tests.cpp", "test/ensemble_config_change_stress.cpp", @@ -2343,12 +2234,6 @@ cc_test( "//src:lib_node_dynamic_demultiplex.so", "//src:lib_node_perform_different_operations.so", "//src:lib_node_choose_maximum.so", - "//src:libcustom_node_east_ocr.so", - "//src:libcustom_node_face_blur.so", - "//src:libcustom_node_model_zoo_intel_object_detection.so", - "//src:libcustom_node_image_transformation.so", - "//src:libcustom_node_add_one.so", - "//src:libcustom_node_horizontal_ocr.so", ] + glob(["test/models_config_json/**"]), deps = [ "optimum-cli", @@ -2366,7 +2251,6 @@ cc_test( ":test_gguf_environment", ":libtest_pythonenvironment", ":network_utils_test", - "//src:custom_nodes_common_buffersqueue", "@com_google_googletest//:gtest", ":libgit2_test", ":pull_hf_model_test", @@ -2414,7 +2298,13 @@ cc_test( }), copts = COPTS_TESTS, local_defines = COMMON_LOCAL_DEFINES, - linkopts = LINKOPTS_ADJUSTED, + linkopts = LINKOPTS_ADJUSTED + select({ + "//conditions:default": [ + "-Wl,-rpath,$$ORIGIN/ovms_test.runfiles/ovms/external/azure/lib", + "-Wl,-rpath,$$ORIGIN/ovms_test.runfiles/azure/lib", + ], + "//src:windows": [], + }), ) cc_library( @@ -2856,17 +2746,6 @@ cc_library( copts = COPTS_TESTS, ) -filegroup( - name = "release_custom_nodes", - srcs = [ - "//src:libcustom_node_add_one.so", - "//src:libcustom_node_east_ocr.so", - "//src:libcustom_node_face_blur.so", - "//src:libcustom_node_horizontal_ocr.so", - "//src:libcustom_node_model_zoo_intel_object_detection.so", - "//src:libcustom_node_image_transformation.so", - ] -) cc_library( name = "openvino_tests", linkstatic = 1, @@ -3008,7 +2887,6 @@ filegroup( name = "static_analysis", srcs = [ "//src:ovms", - "//src:release_custom_nodes", "//src:capi_benchmark", "//src:capi_cpp_example" ] diff --git a/src/filesystem/BUILD b/src/filesystem/BUILD index 2c9c2cebd0..9cd4aedd38 100644 --- a/src/filesystem/BUILD +++ b/src/filesystem/BUILD @@ -81,7 +81,6 @@ ovms_cc_library( ], deps = [ "@azure//:storage", - "@cpprest//:sdk", "libovmsfilesystem", "//src:libovmslogging", "//src:libovmsstring_utils", diff --git a/src/filesystem/azurefilesystem.cpp b/src/filesystem/azurefilesystem.cpp index 66c750afa2..94152be693 100644 --- a/src/filesystem/azurefilesystem.cpp +++ b/src/filesystem/azurefilesystem.cpp @@ -25,63 +25,20 @@ namespace ovms { -static as::cloud_storage_account createDefaultOrAnonymousAccount() { - try { - const char* env_cred = std::getenv("AZURE_STORAGE_CONNECTION_STRING"); +static std::string getConnectionString() { + const char* env_cred = std::getenv("AZURE_STORAGE_CONNECTION_STRING"); - std::string credentials = std::string(_XPLATSTR("DefaultEndpointsProtocol = https;")); - - if (!env_cred) { - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Creating AzureFileSystem anonymous connection string."); - } else { - credentials = std::string(_XPLATSTR(env_cred)); - } - - as::cloud_storage_account storage_account = as::cloud_storage_account::parse(credentials); - if (!storage_account.is_initialized()) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to create default azure storage account"); - throw std::runtime_error("Unable to create default azure storage account"); - } - - const char* use_http = std::getenv("AZURE_STORAGE_USE_HTTP_PROXY"); - - const char* proxy_env; - - std::string https_proxy = std::string(""); - if (!use_http) { - proxy_env = std::getenv("https_proxy"); - } else { - proxy_env = std::getenv("http_proxy"); - } - - if (!proxy_env) { - SPDLOG_LOGGER_DEBUG(azurestorage_logger, "No proxy detected."); - } else { - https_proxy = std::string(proxy_env); - web::web_proxy wproxy(https_proxy); - as::operation_context::set_default_proxy(std::move(wproxy)); - - SPDLOG_LOGGER_DEBUG(azurestorage_logger, "Proxy detected: {}" + https_proxy); - } - - return storage_account; - } catch (const as::storage_exception& e) { - as::request_result result = e.result(); - as::storage_extended_error extended_error = result.extended_error(); - if (!extended_error.message().empty()) { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to create default azure storage account: {}", extended_error.message()); - } else { - SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to create default azure storage account: {}", e.what()); - } - throw e; - } catch (const std::exception& e) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to create default azure storage account: {}", e.what()); - throw e; + if (!env_cred) { + SPDLOG_LOGGER_TRACE(azurestorage_logger, "AZURE_STORAGE_CONNECTION_STRING not set; Azure operations will fail unless using anonymous access."); + return std::string(""); } + + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Using AZURE_STORAGE_CONNECTION_STRING from environment."); + return std::string(env_cred); } AzureFileSystem::AzureFileSystem() : - account_{createDefaultOrAnonymousAccount()} { + connection_string_(getConnectionString()) { SPDLOG_LOGGER_TRACE(azurestorage_logger, "AzureFileSystem default ctor"); } @@ -91,7 +48,7 @@ StatusCode AzureFileSystem::fileExists(const std::string& path, bool* exists) { *exists = false; auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, connection_string_); auto status = azureStorageObj->checkPath(path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, @@ -109,7 +66,7 @@ StatusCode AzureFileSystem::isDirectory(const std::string& path, *is_directory = false; auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, connection_string_); auto status = azureStorageObj->checkPath(path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, @@ -126,7 +83,7 @@ StatusCode AzureFileSystem::fileModificationTime(const std::string& path, int64_t* mtime_ns) { auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, connection_string_); auto status = azureStorageObj->checkPath(path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, @@ -144,7 +101,7 @@ AzureFileSystem::getDirectoryContents(const std::string& path, std::set* contents) { auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, connection_string_); auto status = azureStorageObj->checkPath(path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, @@ -161,7 +118,7 @@ StatusCode AzureFileSystem::getDirectorySubdirs(const std::string& path, std::set* subdirs) { auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, connection_string_); auto status = azureStorageObj->checkPath(path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, @@ -178,7 +135,7 @@ StatusCode AzureFileSystem::getDirectoryFiles(const std::string& path, std::set* files) { auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, connection_string_); auto status = azureStorageObj->checkPath(path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, @@ -195,7 +152,7 @@ StatusCode AzureFileSystem::readTextFile(const std::string& path, std::string* contents) { auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, connection_string_); auto status = azureStorageObj->checkPath(path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, @@ -232,7 +189,7 @@ StatusCode AzureFileSystem::downloadModelVersions(const std::string& path, fs::create_directory(lpath); auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(versionpath, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(versionpath, connection_string_); auto status = azureStorageObj->checkPath(versionpath); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", versionpath, @@ -254,7 +211,7 @@ StatusCode AzureFileSystem::downloadFile(const std::string& remote_path, const std::string& local_path) { auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(remote_path, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(remote_path, connection_string_); auto status = azureStorageObj->checkPath(remote_path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_path, @@ -270,7 +227,7 @@ StatusCode AzureFileSystem::downloadFile(const std::string& remote_path, StatusCode AzureFileSystem::downloadFileFolder(const std::string& path, const std::string& local_path) { auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, connection_string_); auto status = azureStorageObj->checkPath(path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, @@ -287,7 +244,7 @@ StatusCode AzureFileSystem::downloadFileFolderTo(const std::string& path, const std::string& local_path) { auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, connection_string_); auto status = azureStorageObj->checkPath(path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, @@ -302,7 +259,7 @@ StatusCode AzureFileSystem::downloadFileFolderTo(const std::string& path, StatusCode AzureFileSystem::deleteFileFolder(const std::string& path) { auto factory = std::make_shared(); - auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, account_); + auto azureStorageObj = factory.get()->getNewAzureStorageObject(path, connection_string_); auto status = azureStorageObj->checkPath(path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", path, diff --git a/src/filesystem/azurefilesystem.hpp b/src/filesystem/azurefilesystem.hpp index 08c12c3367..d7d3324426 100644 --- a/src/filesystem/azurefilesystem.hpp +++ b/src/filesystem/azurefilesystem.hpp @@ -15,8 +15,6 @@ //***************************************************************************** #pragma once -#define _TURN_OFF_PLATFORM_STRING - #include #include #include @@ -27,8 +25,6 @@ namespace ovms { -namespace as = azure::storage; - class AzureFileSystem : public FileSystem { public: /** @@ -156,7 +152,7 @@ class AzureFileSystem : public FileSystem { * @brief * */ - as::cloud_storage_account account_; + std::string connection_string_; }; } // namespace ovms diff --git a/src/filesystem/azurestorage.cpp b/src/filesystem/azurestorage.cpp index ecda652a56..da4bee8174 100644 --- a/src/filesystem/azurestorage.cpp +++ b/src/filesystem/azurestorage.cpp @@ -15,6 +15,7 @@ //***************************************************************************** #include "azurestorage.hpp" +#include #include #include @@ -23,18 +24,13 @@ namespace ovms { -using namespace utility; - const std::string UNAVAILABLE_PATH_ERROR = "Unable to access path: {}"; -const std::string AzureStorageAdapter::extractAzureStorageExceptionMessage(const as::storage_exception& e) { - as::request_result result = e.result(); - as::storage_extended_error extended_error = result.extended_error(); - if (!extended_error.message().empty()) { - return extended_error.message(); - } else { - return e.what(); +const std::string AzureStorageAdapter::extractAzureStorageExceptionMessage(const Azure::Storage::StorageException& e) { + if (!e.Message.empty()) { + return e.Message; } + return e.what(); } StatusCode AzureStorageAdapter::CreateLocalDir(const std::string& path) { @@ -52,10 +48,14 @@ bool AzureStorageAdapter::isAbsolutePath(const std::string& path) { return !path.empty() && (path[0] == '/'); } -AzureStorageBlob::AzureStorageBlob(const std::string& path, as::cloud_storage_account& account) { - account_ = account; - as_blob_client_ = account_.create_cloud_blob_client(); - isPathValidationOk_ = false; +// ======================================================================== +// AzureStorageBlob +// ======================================================================== + +AzureStorageBlob::AzureStorageBlob(const std::string& path, const std::string& connection_string) : + isPathValidationOk_(false), + connection_string_(connection_string), + as_container_(asblobs::BlobContainerClient("https://placeholder.blob.core.windows.net/placeholder")) { } StatusCode AzureStorageBlob::checkPath(const std::string& path) { @@ -72,22 +72,25 @@ StatusCode AzureStorageBlob::checkPath(const std::string& path) { return status; } - as_container_ = as_blob_client_.get_container_reference(container_); + as_container_ = asblobs::BlobContainerClient::CreateFromConnectionString(connection_string_, container_); - if (!as_container_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Container does not exist: {} -> {}", fullPath_, container_); - return StatusCode::AS_CONTAINER_NOT_FOUND; + try { + as_container_.GetProperties(); + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Container does not exist: {} -> {}", fullPath_, container_); + return StatusCode::AS_CONTAINER_NOT_FOUND; + } + throw; } isPathValidationOk_ = true; - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -100,20 +103,23 @@ StatusCode AzureStorageBlob::fileExists(bool* exists) { return status; } - as_blob_ = as_container_.get_blob_reference(blockpath_); - if (!as_blob_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); - return StatusCode::AS_FILE_NOT_FOUND; + auto blob_client = as_container_.GetBlobClient(blockpath_); + try { + blob_client.GetProperties(); + *exists = true; + return StatusCode::OK; + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + throw; } - - *exists = true; - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -126,40 +132,24 @@ StatusCode AzureStorageBlob::isDirectory(bool* is_directory) { return status; } - as::cloud_blob_directory temp_directory = as_container_.get_directory_reference(blockpath_); - as::cloud_blob_directory parent_directory = temp_directory.get_parent_reference(); - - // List blobs in the blob container - as::continuation_token token; - do { - as::list_blob_item_segment result; - // Check if we are at container root - if (parent_directory.prefix() == "") - result = as_container_.list_blobs_segmented(token); - else - result = parent_directory.list_blobs_segmented(token); - - for (auto& item : result.results()) { - if (!item.is_blob()) { - std::string prefix = item.as_directory().prefix(); - if (prefix.back() == '/') - prefix.pop_back(); - if (prefix == blockpath_) { - *is_directory = true; - } - } - } - - token = result.continuation_token(); - } while (!token.empty()); + if (blockpath_.empty()) { + // Container root is always a directory when the container exists + *is_directory = true; + return StatusCode::OK; + } + // A virtual directory exists if any blobs share the prefix + Azure::Storage::Blobs::ListBlobsOptions options; + options.Prefix = blockpath_ + "/"; + options.PageSizeHint = 1; + auto page = as_container_.ListBlobsByHierarchy("/", options); + *is_directory = !page.Blobs.empty() || !page.BlobPrefixes.empty(); return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -171,27 +161,26 @@ StatusCode AzureStorageBlob::fileModificationTime(int64_t* mtime_ns) { return status; } - as_blob_ = as_container_.get_blob_reference(blockpath_); - if (!as_blob_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); - return StatusCode::AS_FILE_NOT_FOUND; + auto blob_client = as_container_.GetBlobClient(blockpath_); + try { + auto props = blob_client.GetProperties(); + auto tp = static_cast(props.Value.LastModified); + auto nanoseconds = std::chrono::duration_cast(tp.time_since_epoch()).count(); + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Modification time for {} is {}", fullPath_, nanoseconds); + *mtime_ns = nanoseconds; + return StatusCode::OK; + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + throw; } - - as::cloud_blob_properties& prop = as_blob_.properties(); - utility::datetime time = prop.last_modified(); - std::string date = time.to_string(); - - auto nanoseconds = time.to_interval(); - - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Modification time for {} is {}", fullPath_, nanoseconds); - *mtime_ns = nanoseconds; - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -203,34 +192,24 @@ StatusCode AzureStorageBlob::getDirectoryContents(files_list_t* contents) { return status; } - as::cloud_blob_directory parent_directory = as_container_.get_directory_reference(blockpath_); - - // List blobs in the blob container - as::continuation_token token; - do { - as::list_blob_item_segment result; - // Check if we are at container root - if (parent_directory.prefix() == "") - result = as_container_.list_blobs_segmented(token); - else - result = parent_directory.list_blobs_segmented(token); - - for (auto& item : result.results()) { - if (item.is_blob()) { - contents->insert(getLastPathPart(item.as_blob().name())); - } else { - contents->insert(getLastPathPart(item.as_directory().prefix())); - } + Azure::Storage::Blobs::ListBlobsOptions options; + if (!blockpath_.empty()) { + options.Prefix = blockpath_ + "/"; + } + for (auto page = as_container_.ListBlobsByHierarchy("/", options); page.HasPage(); page.MoveToNextPage()) { + for (const auto& blob : page.Blobs) { + contents->insert(getLastPathPart(blob.Name)); } - } while (!token.empty()); - + for (const auto& prefix : page.BlobPrefixes) { + contents->insert(getLastPathPart(prefix)); + } + } return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -242,32 +221,21 @@ StatusCode AzureStorageBlob::getDirectorySubdirs(files_list_t* subdirs) { return status; } - as::cloud_blob_directory parent_directory = as_container_.get_directory_reference(blockpath_); - - // List blobs in the blob container - as::continuation_token token; - do { - as::list_blob_item_segment result; - // Check if we are at container root - if (parent_directory.prefix() == "") - result = as_container_.list_blobs_segmented(token); - else - result = parent_directory.list_blobs_segmented(token); - - for (auto& item : result.results()) { - if (!item.is_blob()) { - subdirs->insert(getLastPathPart(item.as_directory().prefix())); - } + Azure::Storage::Blobs::ListBlobsOptions options; + if (!blockpath_.empty()) { + options.Prefix = blockpath_ + "/"; + } + for (auto page = as_container_.ListBlobsByHierarchy("/", options); page.HasPage(); page.MoveToNextPage()) { + for (const auto& prefix : page.BlobPrefixes) { + subdirs->insert(getLastPathPart(prefix)); } - } while (!token.empty()); - + } return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -279,32 +247,21 @@ StatusCode AzureStorageBlob::getDirectoryFiles(files_list_t* files) { return status; } - as::cloud_blob_directory parent_directory = as_container_.get_directory_reference(blockpath_); - - // List blobs in the blob container - as::continuation_token token; - do { - as::list_blob_item_segment result; - // Check if we are at container root - if (parent_directory.prefix() == "") - result = as_container_.list_blobs_segmented(token); - else - result = parent_directory.list_blobs_segmented(token); - - for (auto& item : result.results()) { - if (item.is_blob()) { - files->insert(getLastPathPart(item.as_blob().name())); - } + Azure::Storage::Blobs::ListBlobsOptions options; + if (!blockpath_.empty()) { + options.Prefix = blockpath_ + "/"; + } + for (auto page = as_container_.ListBlobsByHierarchy("/", options); page.HasPage(); page.MoveToNextPage()) { + for (const auto& blob : page.Blobs) { + files->insert(getLastPathPart(blob.Name)); } - } while (!token.empty()); - + } return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -316,26 +273,24 @@ StatusCode AzureStorageBlob::readTextFile(std::string* contents) { return status; } - as_blob_ = as_container_.get_blob_reference(blockpath_); - if (!as_blob_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Nlock blob does not exist: {} -> {}", fullPath_, blockpath_); - return StatusCode::AS_FILE_NOT_FOUND; + auto blob_client = as_container_.GetBlockBlobClient(blockpath_); + try { + auto result = blob_client.Download(); + auto body_bytes = result.Value.BodyStream->ReadToEnd(); + *contents = std::string(body_bytes.begin(), body_bytes.end()); + return StatusCode::OK; + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + throw; } - - as_block_blob_ = as_container_.get_block_blob_reference(blockpath_); - - concurrency::streams::container_buffer> buffer; - concurrency::streams::ostream output_stream(buffer); - as_block_blob_.download_to_stream(output_stream); - *contents = utility::string_t(buffer.collection().begin(), buffer.collection().end()); - - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -360,20 +315,22 @@ StatusCode AzureStorageBlob::deleteFileFolder() { return status; } - as_blob_ = as_container_.get_blob_reference(blockpath_); - if (!as_blob_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "block blob does not exist: {} -> {}", fullPath_, blockpath_); - return StatusCode::AS_FILE_NOT_FOUND; + auto blob_client = as_container_.GetBlobClient(blockpath_); + try { + blob_client.Delete(); + return StatusCode::OK; + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "block blob does not exist: {} -> {}", fullPath_, blockpath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + throw; } - - as_blob_.delete_blob(); - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -396,20 +353,22 @@ StatusCode AzureStorageBlob::downloadFile(const std::string& local_path) { return status; } - as_blob_ = as_container_.get_blob_reference(blockpath_); - if (!as_blob_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); - return StatusCode::AS_FILE_NOT_FOUND; + auto blob_client = as_container_.GetBlobClient(blockpath_); + try { + blob_client.DownloadTo(local_path); + return StatusCode::OK; + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Block blob does not exist: {} -> {}", fullPath_, blockpath_); + return StatusCode::AS_FILE_NOT_FOUND; + } + throw; } - - as_blob_.download_to_file(local_path); - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -453,7 +412,7 @@ StatusCode AzureStorageBlob::downloadFileFolderTo(const std::string& local_path) local_dir_path); auto factory = std::make_shared(); - auto azureSubdirStorageObj = factory.get()->getNewAzureStorageObject(remote_dir_path, account_); + auto azureSubdirStorageObj = factory.get()->getNewAzureStorageObject(remote_dir_path, connection_string_); status = azureSubdirStorageObj->checkPath(remote_dir_path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_dir_path, @@ -465,8 +424,7 @@ StatusCode AzureStorageBlob::downloadFileFolderTo(const std::string& local_path) if (mkdir_status != StatusCode::OK) { return status; } - auto download_dir_status = - azureSubdirStorageObj->downloadFileFolderTo(local_dir_path); + auto download_dir_status = azureSubdirStorageObj->downloadFileFolderTo(local_dir_path); if (download_dir_status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to download directory from {} to {}", remote_dir_path, local_dir_path); @@ -481,7 +439,7 @@ StatusCode AzureStorageBlob::downloadFileFolderTo(const std::string& local_path) local_file_path); auto factory = std::make_shared(); - auto azureFiledirStorageObj = factory.get()->getNewAzureStorageObject(remote_file_path, account_); + auto azureFiledirStorageObj = factory.get()->getNewAzureStorageObject(remote_file_path, connection_string_); status = azureFiledirStorageObj->checkPath(remote_file_path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to download directory from {} to {}", @@ -489,8 +447,7 @@ StatusCode AzureStorageBlob::downloadFileFolderTo(const std::string& local_path) return status; } - auto download_status = - azureFiledirStorageObj->downloadFile(local_file_path); + auto download_status = azureFiledirStorageObj->downloadFile(local_file_path); if (download_status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to save file from {} to {}", remote_file_path, local_file_path); @@ -498,12 +455,11 @@ StatusCode AzureStorageBlob::downloadFileFolderTo(const std::string& local_path) } } return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -550,13 +506,9 @@ StatusCode AzureStorageBlob::parseFilePath(const std::string& path) { subdirs_ = FindSubdirectories(fullPath_); - std::set subdirs; - if (share_end > share_start) { container_ = path.substr(share_start, share_end - share_start); - blockpath_ = path.substr(share_end + 1, file_end - share_end - 1); - } else { // No directory and no file container_ = path.substr(share_start); @@ -571,10 +523,16 @@ StatusCode AzureStorageBlob::parseFilePath(const std::string& path) { return StatusCode::OK; } -AzureStorageFile::AzureStorageFile(const std::string& path, as::cloud_storage_account& account) { - account_ = account; - as_file_client_ = account_.create_cloud_file_client(); - isPathValidationOk_ = false; +// ======================================================================== +// AzureStorageFile +// ======================================================================== + +AzureStorageFile::AzureStorageFile(const std::string& path, const std::string& connection_string) : + isPathValidationOk_(false), + connection_string_(connection_string), + as_share_(asfiles::ShareClient("https://placeholder.file.core.windows.net/placeholder")), + as_directory_(asfiles::ShareDirectoryClient("https://placeholder.file.core.windows.net/placeholder/dir")), + as_file1_(asfiles::ShareFileClient("https://placeholder.file.core.windows.net/placeholder/dir/file")) { } StatusCode AzureStorageFile::checkPath(const std::string& path) { @@ -591,12 +549,16 @@ StatusCode AzureStorageFile::checkPath(const std::string& path) { return status; } - as_file_client_ = account_.create_cloud_file_client(); - as_share_ = as_file_client_.get_share_reference(share_); + as_share_ = asfiles::ShareClient::CreateFromConnectionString(connection_string_, share_); - if (!as_share_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Share does not exist: {} -> {}", path, share_); - return StatusCode::AS_SHARE_NOT_FOUND; + try { + as_share_.GetProperties(); + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Share does not exist: {} -> {}", path, share_); + return StatusCode::AS_SHARE_NOT_FOUND; + } + throw; } if (directory_.empty()) { @@ -605,17 +567,35 @@ StatusCode AzureStorageFile::checkPath(const std::string& path) { } isPathValidationOk_ = true; - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } +static asfiles::ShareDirectoryClient getDeepestExistingDirectory( + const asfiles::ShareClient& share, + const std::vector& subdirs) { + auto dir_client = share.GetRootDirectoryClient(); + std::string tmp_dir = ""; + for (const auto& segment : subdirs) { + if (segment.empty()) + continue; + tmp_dir = tmp_dir.empty() ? segment : (tmp_dir + "/" + segment); + try { + auto sub = share.GetRootDirectoryClient().GetSubdirectoryClient(tmp_dir); + sub.GetProperties(); + dir_client = sub; + } catch (const Azure::Storage::StorageException&) { + break; + } + } + return dir_client; +} + StatusCode AzureStorageFile::fileExists(bool* exists) { try { *exists = false; @@ -625,37 +605,24 @@ StatusCode AzureStorageFile::fileExists(bool* exists) { return status; } - as::cloud_file_directory as_last_working_subdir; - - std::string tmp_dir = ""; - + auto dir_client = getDeepestExistingDirectory(as_share_, subdirs_); + auto file_client = dir_client.GetFileClient(file_); try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; + file_client.GetProperties(); + *exists = true; + return StatusCode::OK; + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); + return StatusCode::AS_FILE_NOT_FOUND; } - } catch (const as::storage_exception& e) { - } - - as_file1_ = as_last_working_subdir.get_file_reference(file_); - if (!as_file1_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); - return StatusCode::AS_FILE_NOT_FOUND; + throw; } - - *exists = true; - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -669,21 +636,21 @@ StatusCode AzureStorageFile::isDirectory(bool* is_directory) { } std::string tmp_dir = ""; - - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - return StatusCode::OK; - } + for (const auto& segment : subdirs_) { + if (segment.empty()) + continue; + tmp_dir = tmp_dir.empty() ? segment : (tmp_dir + "/" + segment); + try { + auto sub = as_share_.GetRootDirectoryClient().GetSubdirectoryClient(tmp_dir); + sub.GetProperties(); + } catch (const Azure::Storage::StorageException&) { + return StatusCode::OK; } - - *is_directory = true; - return StatusCode::OK; - } catch (const as::storage_exception& e) { } - } catch (const as::storage_exception& e) { + + *is_directory = true; + return StatusCode::OK; + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); @@ -699,33 +666,37 @@ StatusCode AzureStorageFile::fileModificationTime(int64_t* mtime_ns) { return status; } - as_directory_ = as_share_.get_directory_reference(_XPLATSTR(directory_)); - if (!as_directory_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "Directory does not exist: {} -> {}", fullPath_, directory_); - return StatusCode::AS_FILE_NOT_FOUND; + as_directory_ = as_share_.GetRootDirectoryClient().GetSubdirectoryClient(directory_); + try { + as_directory_.GetProperties(); + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "Directory does not exist: {} -> {}", fullPath_, directory_); + return StatusCode::AS_FILE_NOT_FOUND; + } + throw; } - as_file1_ = as_directory_.get_file_reference(_XPLATSTR(file_)); - if (!as_file1_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); - return StatusCode::AS_FILE_NOT_FOUND; + as_file1_ = as_directory_.GetFileClient(file_); + try { + auto props = as_file1_.GetProperties(); + auto tp = static_cast(props.Value.LastModified); + auto nanoseconds = std::chrono::duration_cast(tp.time_since_epoch()).count(); + SPDLOG_LOGGER_TRACE(azurestorage_logger, "Modification time for {} is {}", fullPath_, nanoseconds); + *mtime_ns = nanoseconds; + return StatusCode::OK; + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); + return StatusCode::AS_FILE_NOT_FOUND; + } + throw; } - - as::cloud_file_properties& prop = as_file1_.properties(); - utility::datetime time = prop.last_modified(); - std::string date = time.to_string(); - - auto nanoseconds = time.to_interval(); - - SPDLOG_LOGGER_TRACE(azurestorage_logger, "Modification time for {} is {}", fullPath_, nanoseconds); - *mtime_ns = nanoseconds; - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -737,43 +708,22 @@ StatusCode AzureStorageFile::getDirectoryContents(files_list_t* contents) { return status; } - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; + auto dir_client = getDeepestExistingDirectory(as_share_, subdirs_); - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; + for (auto page = dir_client.ListFilesAndDirectories(); page.HasPage(); page.MoveToNextPage()) { + for (const auto& file : page.Files) { + contents->insert(file.Name); } - } catch (const as::storage_exception& e) { - } - - // List files and directories in the directory - as::continuation_token token; - do { - as::list_file_and_directory_result_segment result = as_last_working_subdir.list_files_and_directories_segmented(token); - for (auto& item : result.results()) { - if (item.is_file()) { - contents->insert(item.as_file().name()); - } - if (item.is_directory()) { - contents->insert(item.as_directory().name()); - } + for (const auto& dir : page.Directories) { + contents->insert(dir.Name); } - } while (!token.empty()); - + } return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -785,40 +735,19 @@ StatusCode AzureStorageFile::getDirectorySubdirs(files_list_t* subdirs) { return status; } - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; + auto dir_client = getDeepestExistingDirectory(as_share_, subdirs_); - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; + for (auto page = dir_client.ListFilesAndDirectories(); page.HasPage(); page.MoveToNextPage()) { + for (const auto& dir : page.Directories) { + subdirs->insert(dir.Name); } - } catch (const as::storage_exception& e) { } - - // List files and directories in the directory - as::continuation_token token; - do { - as::list_file_and_directory_result_segment result = as_last_working_subdir.list_files_and_directories_segmented(token); - for (auto& item : result.results()) { - if (item.is_directory()) { - subdirs->insert(item.as_directory().name()); - } - } - } while (!token.empty()); - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -830,40 +759,19 @@ StatusCode AzureStorageFile::getDirectoryFiles(files_list_t* files) { return status; } - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; + auto dir_client = getDeepestExistingDirectory(as_share_, subdirs_); - try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; + for (auto page = dir_client.ListFilesAndDirectories(); page.HasPage(); page.MoveToNextPage()) { + for (const auto& file : page.Files) { + files->insert(file.Name); } - } catch (const as::storage_exception& e) { } - - // List files and directories in the directory - as::continuation_token token; - do { - as::list_file_and_directory_result_segment result = as_last_working_subdir.list_files_and_directories_segmented(token); - for (auto& item : result.results()) { - if (item.is_file()) { - files->insert(item.as_file().name()); - } - } - } while (!token.empty()); - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -875,40 +783,25 @@ StatusCode AzureStorageFile::readTextFile(std::string* contents) { return status; } - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; - + auto dir_client = getDeepestExistingDirectory(as_share_, subdirs_); + as_file1_ = dir_client.GetFileClient(file_); try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; + auto result = as_file1_.Download(); + auto body_bytes = result.Value.BodyStream->ReadToEnd(); + *contents = std::string(body_bytes.begin(), body_bytes.end()); + return StatusCode::OK; + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); + return StatusCode::AS_FILE_NOT_FOUND; } - } catch (const as::storage_exception& e) { + throw; } - - as_file1_ = as_last_working_subdir.get_file_reference(_XPLATSTR(file_)); - if (!as_file1_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - concurrency::streams::container_buffer> buffer; - concurrency::streams::ostream output_stream(buffer); - as_file1_.download_to_stream(output_stream); - *contents = utility::string_t(buffer.collection().begin(), buffer.collection().end()); - - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -933,36 +826,23 @@ StatusCode AzureStorageFile::deleteFileFolder() { return status; } - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; - + auto dir_client = getDeepestExistingDirectory(as_share_, subdirs_); + as_file1_ = dir_client.GetFileClient(file_); try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; + as_file1_.Delete(); + return StatusCode::OK; + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); + return StatusCode::AS_FILE_NOT_FOUND; } - } catch (const as::storage_exception& e) { + throw; } - - as_file1_ = as_last_working_subdir.get_file_reference(_XPLATSTR(file_)); - if (!as_file1_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); - return StatusCode::AS_FILE_NOT_FOUND; - } - - as_file1_.delete_file(); - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -974,36 +854,23 @@ StatusCode AzureStorageFile::downloadFile(const std::string& local_path) { return status; } - as::cloud_file_directory as_last_working_subdir; - std::string tmp_dir = ""; - + auto dir_client = getDeepestExistingDirectory(as_share_, subdirs_); + as_file1_ = dir_client.GetFileClient(file_); try { - for (std::vector::size_type i = 0; i != subdirs_.size(); i++) { - tmp_dir = tmp_dir + (i == 0 ? "" : "/") + subdirs_[i]; - as::cloud_file_directory as_tmp_subdir = as_share_.get_directory_reference(tmp_dir); - if (!as_tmp_subdir.exists()) { - break; - } - - as_last_working_subdir = as_tmp_subdir; + as_file1_.DownloadTo(local_path); + return StatusCode::OK; + } catch (const Azure::Storage::StorageException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); + return StatusCode::AS_FILE_NOT_FOUND; } - } catch (const as::storage_exception& e) { - } - - as_file1_ = as_last_working_subdir.get_file_reference(_XPLATSTR(file_)); - if (!as_file1_.exists()) { - SPDLOG_LOGGER_WARN(azurestorage_logger, "File does not exist: {} -> {}", fullPath_, file_); - return StatusCode::AS_FILE_NOT_FOUND; + throw; } - - as_file1_.download_to_file(local_path); - return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -1047,8 +914,8 @@ StatusCode AzureStorageFile::downloadFileFolderTo(const std::string& local_path) local_dir_path); auto factory = std::make_shared(); - auto azureSubdirStorageObj = factory.get()->getNewAzureStorageObject(remote_dir_path, account_); - auto status = azureSubdirStorageObj->checkPath(remote_dir_path); + auto azureSubdirStorageObj = factory.get()->getNewAzureStorageObject(remote_dir_path, connection_string_); + status = azureSubdirStorageObj->checkPath(remote_dir_path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_dir_path, ovms::Status(status).string()); @@ -1059,8 +926,7 @@ StatusCode AzureStorageFile::downloadFileFolderTo(const std::string& local_path) if (mkdir_status != StatusCode::OK) { return status; } - auto download_dir_status = - azureSubdirStorageObj->downloadFileFolderTo(local_dir_path); + auto download_dir_status = azureSubdirStorageObj->downloadFileFolderTo(local_dir_path); if (download_dir_status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to download directory from {} to {}", remote_dir_path, local_dir_path); @@ -1075,16 +941,15 @@ StatusCode AzureStorageFile::downloadFileFolderTo(const std::string& local_path) local_file_path); auto factory = std::make_shared(); - auto azureFileStorageObj = factory.get()->getNewAzureStorageObject(remote_file_path, account_); - auto status = azureFileStorageObj->checkPath(remote_file_path); + auto azureFileStorageObj = factory.get()->getNewAzureStorageObject(remote_file_path, connection_string_); + status = azureFileStorageObj->checkPath(remote_file_path); if (status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Check path failed: {} -> {}", remote_file_path, ovms::Status(status).string()); return status; } - auto download_status = - azureFileStorageObj->downloadFile(local_file_path); + auto download_status = azureFileStorageObj->downloadFile(local_file_path); if (download_status != StatusCode::OK) { SPDLOG_LOGGER_WARN(azurestorage_logger, "Unable to save file from {} to {}", remote_file_path, local_file_path); @@ -1092,12 +957,11 @@ StatusCode AzureStorageFile::downloadFileFolderTo(const std::string& local_path) } } return StatusCode::OK; - } catch (const as::storage_exception& e) { + } catch (const Azure::Storage::StorageException& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, "Unable to access path: {}", extractAzureStorageExceptionMessage(e)); } catch (const std::exception& e) { SPDLOG_LOGGER_ERROR(azurestorage_logger, UNAVAILABLE_PATH_ERROR, e.what()); } - return StatusCode::AS_FILE_NOT_FOUND; } @@ -1108,9 +972,7 @@ std::vector AzureStorageAdapter::FindSubdirectories(std::string pat while ((pos = path.find('/', pos)) != std::string::npos) { std::string substring(path.substr(prev_pos, pos - prev_pos)); - output.push_back(substring); - prev_pos = ++pos; } @@ -1152,7 +1014,6 @@ StatusCode AzureStorageFile::parseFilePath(const std::string& path) { if (share_end > share_start) { share_ = path.substr(share_start, share_end - share_start); - directory_ = path.substr(share_end + 1, file_start - share_end - 1); // No file or no directory @@ -1181,11 +1042,11 @@ StatusCode AzureStorageFile::parseFilePath(const std::string& path) { return StatusCode::OK; } -std::shared_ptr AzureStorageFactory::getNewAzureStorageObject(const std::string& path, as::cloud_storage_account& account) { +std::shared_ptr AzureStorageFactory::getNewAzureStorageObject(const std::string& path, const std::string& connection_string) { if (isBlobStoragePath(path)) - return std::make_shared(path, account); + return std::make_shared(path, connection_string); - return std::make_shared(path, account); + return std::make_shared(path, connection_string); } bool AzureStorageFactory::isBlobStoragePath(std::string path) { diff --git a/src/filesystem/azurestorage.hpp b/src/filesystem/azurestorage.hpp index ea0eb0e33a..54222f54fb 100644 --- a/src/filesystem/azurestorage.hpp +++ b/src/filesystem/azurestorage.hpp @@ -22,21 +22,21 @@ #include #include "src/status.hpp" + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wall" #pragma GCC diagnostic ignored "-Wunknown-pragmas" #pragma GCC diagnostic ignored "-Wreorder" #pragma GCC diagnostic ignored "-Wunused-value" -#include - -#include "was/blob.h" -#include "was/common.h" -#include "was/storage_account.h" +#include +#include #pragma GCC diagnostic pop namespace ovms { -namespace as = azure::storage; +namespace asblobs = Azure::Storage::Blobs; +namespace asfiles = Azure::Storage::Files::Shares; + using files_list_t = std::set; class AzureStorageAdapter { @@ -62,7 +62,7 @@ class AzureStorageAdapter { virtual ~AzureStorageAdapter() = default; protected: - const std::string extractAzureStorageExceptionMessage(const as::storage_exception& e); + const std::string extractAzureStorageExceptionMessage(const Azure::Storage::StorageException& e); private: virtual StatusCode parseFilePath(const std::string& path) = 0; @@ -70,7 +70,7 @@ class AzureStorageAdapter { class AzureStorageBlob : public AzureStorageAdapter { public: - AzureStorageBlob(const std::string& path, as::cloud_storage_account& account); + AzureStorageBlob(const std::string& path, const std::string& connection_string); StatusCode checkPath(const std::string& path) override; @@ -115,20 +115,14 @@ class AzureStorageBlob : public AzureStorageAdapter { std::string container_; - as::cloud_blob_container as_container_; - - as::cloud_block_blob as_block_blob_; + std::string connection_string_; - as::cloud_blob as_blob_; - - as::cloud_storage_account account_; - - as::cloud_blob_client as_blob_client_; + asblobs::BlobContainerClient as_container_; }; class AzureStorageFile : public AzureStorageAdapter { public: - AzureStorageFile(const std::string& path, as::cloud_storage_account& account); + AzureStorageFile(const std::string& path, const std::string& connection_string); StatusCode checkPath(const std::string& path) override; @@ -171,20 +165,18 @@ class AzureStorageFile : public AzureStorageAdapter { std::string share_; - as::cloud_storage_account account_; - - as::cloud_file_client as_file_client_; + std::string connection_string_; - as::cloud_file_share as_share_; + asfiles::ShareClient as_share_; - as::cloud_file_directory as_directory_; + asfiles::ShareDirectoryClient as_directory_; - as::cloud_file as_file1_; + asfiles::ShareFileClient as_file1_; }; class AzureStorageFactory { public: - std::shared_ptr getNewAzureStorageObject(const std::string& path, as::cloud_storage_account& account); + std::shared_ptr getNewAzureStorageObject(const std::string& path, const std::string& connection_string); private: bool isBlobStoragePath(std::string path); diff --git a/src/test/c_api_stress_tests.cpp b/src/test/c_api_stress_tests.cpp index c247ddc96c..79cca10323 100644 --- a/src/test/c_api_stress_tests.cpp +++ b/src/test/c_api_stress_tests.cpp @@ -84,73 +84,6 @@ static const char* stressTestPipelineOneDummyConfigSpecificVersionUsed = R"( ] })"; -static const char* stressPipelineCustomNodeAddOneThenDummy = R"( -{ - "custom_node_library_config_list": [ - { - "name": "lib_add_one", - "base_path": "/ovms/bazel-bin/src/libcustom_node_add_one.so" - } - ], - "model_config_list": [ - { - "config": { - "name": "dummy", - "base_path": "/ovms/src/test/dummy", - "target_device": "CPU", - "model_version_policy": {"all": {}}, - "nireq": 20, - "shape": {"b": "(1,10) "} - } - } - ], - "pipeline_config_list": [ - { - "name": "pipeline1Dummy", - "inputs": ["custom_dummy_input"], - "nodes": [ - { - "name": "custom_node", - "library_name": "lib_add_one", - "type": "custom", - "params": { - "output_queue_size": "20", - "info_queue_size": "20", - "add_number": "1", - "sub_number": "0" - }, - "inputs": [ - {"input_numbers": {"node_name": "request", - "data_item": "custom_dummy_input"}} - ], - "outputs": [ - {"data_item": "output_numbers", - "alias": "custom_node_output"} - ] - }, - { - "name": "dummyNode", - "model_name": "dummy", - "type": "DL model", - "inputs": [ - {"b": {"node_name": "custom_node", - "data_item": "custom_node_output"}} - ], - "outputs": [ - {"data_item": "a", - "alias": "dummy_output"} - ] - } - ], - "outputs": [ - {"custom_dummy_output": {"node_name": "dummyNode", - "data_item": "dummy_output"} - } - ] - } - ] -})"; - using testing::_; using testing::Return; @@ -467,169 +400,3 @@ TEST_F(StressCapiConfigChanges, AddModelDuringGetModelStatusLoad) { requiredLoadResults, allowedLoadResults); } - -class StressPipelineCustomNodesWithPreallocatedBuffersCapiConfigChanges : public StressCapiConfigChanges { -public: - void checkInferResponse(OVMS_InferenceResponse* response, std::string& expectedOutputName) override { - ASSERT_NE(response, nullptr); - uint32_t outputCount = 42; - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutputCount(response, &outputCount)); - ASSERT_EQ(outputCount, 1); - const void* voutputData = nullptr; - size_t bytesize = 42; - uint32_t outputId = 0; - OVMS_DataType datatype = (OVMS_DataType)199; - const int64_t* shape{nullptr}; - size_t dimCount = 42; - OVMS_BufferType bufferType = (OVMS_BufferType)199; - uint32_t deviceId = 42; - const char* outputName{nullptr}; - ASSERT_CAPI_STATUS_NULL(OVMS_InferenceResponseOutput(response, outputId, &outputName, &datatype, &shape, &dimCount, &voutputData, &bytesize, &bufferType, &deviceId)); - ASSERT_EQ(std::string("custom_dummy_output"), outputName); - EXPECT_EQ(datatype, OVMS_DATATYPE_FP32); - EXPECT_EQ(dimCount, 2); - EXPECT_EQ(bufferType, OVMS_BUFFERTYPE_CPU); - EXPECT_EQ(deviceId, 0); - for (size_t i = 0; i < DUMMY_MODEL_SHAPE.size(); ++i) { - EXPECT_EQ(DUMMY_MODEL_SHAPE[i], shape[i]) << "Different at:" << i << " place."; - } - const float* outputData = reinterpret_cast(voutputData); - ASSERT_EQ(bytesize, sizeof(float) * DUMMY_MODEL_INPUT_SIZE); - std::array data{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - for (size_t i = 0; i < data.size(); ++i) { - EXPECT_EQ(data[i] + 2, outputData[i]) << "Different at:" << i << " place."; - } - } -}; - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersCapiConfigChanges, RemoveCustomLibraryDuringPredictLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK, - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; - std::set allowedLoadResults = {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE}; - performStressTest( - &StressCapiConfigChanges::triggerCApiInferenceInALoop, - &StressCapiConfigChanges::removePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersCapiConfigChanges, RenameCustomLibraryDuringPredictLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &StressCapiConfigChanges::triggerCApiInferenceInALoop, - &StressCapiConfigChanges::renamePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersCapiConfigChanges, ChangeParamCustomLibraryDuringPredictLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &StressCapiConfigChanges::triggerCApiInferenceInALoop, - &StressCapiConfigChanges::changeParamPreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersCapiConfigChanges, ReduceQueueSizeCustomLibraryDuringPredictLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &StressCapiConfigChanges::triggerCApiInferenceInALoop, - &StressCapiConfigChanges::reduceQueueSizePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersCapiConfigChanges, IncreaseQueueSizeCustomLibraryDuringPredictLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &StressCapiConfigChanges::triggerCApiInferenceInALoop, - &StressCapiConfigChanges::increaseQueueSizePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersCapiConfigChanges, RemoveCustomLibraryDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK, // we expect full continuity of operation - StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // we hit when all config changes finish to propagate - std::set allowedLoadResults = {}; - performStressTest( - &StressCapiConfigChanges::triggerCApiGetMetadataInALoop, - &StressCapiConfigChanges::removePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersCapiConfigChanges, RenameCustomLibraryDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &StressCapiConfigChanges::triggerCApiGetMetadataInALoop, - &StressCapiConfigChanges::renamePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersCapiConfigChanges, ChangeParamCustomLibraryDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &StressCapiConfigChanges::triggerCApiGetMetadataInALoop, - &StressCapiConfigChanges::changeParamPreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersCapiConfigChanges, ReduceQueueSizeCustomLibraryDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &StressCapiConfigChanges::triggerCApiGetMetadataInALoop, - &StressCapiConfigChanges::reduceQueueSizePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} - -TEST_F(StressPipelineCustomNodesWithPreallocatedBuffersCapiConfigChanges, IncreaseQueueSizeCustomLibraryDuringGetMetadataLoad) { - SetUpConfig(stressPipelineCustomNodeAddOneThenDummy); - bool performWholeConfigReload = true; - std::set requiredLoadResults = {StatusCode::OK}; // we expect full continuity of operation - std::set allowedLoadResults = {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET}; // might hit reload phase - performStressTest( - &StressCapiConfigChanges::triggerCApiGetMetadataInALoop, - &StressCapiConfigChanges::increaseQueueSizePreallocatedCustomLibraryUsed, - performWholeConfigReload, - requiredLoadResults, - allowedLoadResults); -} diff --git a/tests/file_lists/lib_files.txt b/tests/file_lists/lib_files.txt index 12edf4b01e..0d8f9aa95f 100644 --- a/tests/file_lists/lib_files.txt +++ b/tests/file_lists/lib_files.txt @@ -1,13 +1,15 @@ -custom_nodes libOpenCL.so->libOpenCL.so.1.0.0 libOpenCL.so.1->libOpenCL.so.1.0.0 libOpenCL.so.1.0.0 -libazurestorage.so -libazurestorage.so.7->libazurestorage.so -libazurestorage.so.7.5->libazurestorage.so -libcpprest.so -libcpprest.so.2.10->libcpprest.so +libazure-core.so->libazure-core.so.1.17.0-beta.1 +libazure-core.so.1.17.0-beta.1 +libazure-storage-blobs.so->libazure-storage-blobs.so.12.18.0 +libazure-storage-blobs.so.12.18.0 +libazure-storage-common.so->libazure-storage-common.so.12.14.0 +libazure-storage-common.so.12.14.0 +libazure-storage-files-shares.so->libazure-storage-files-shares.so.12.18.0 +libazure-storage-files-shares.so.12.18.0 libespeak-ng.so->libespeak-ng.so.1 libespeak-ng.so.1->libespeak-ng.so.1.52.0.1 libespeak-ng.so.1.52.0.1 diff --git a/tests/file_lists/lib_files_python.txt b/tests/file_lists/lib_files_python.txt index 0f9320eaaa..6a4cf1c0ad 100644 --- a/tests/file_lists/lib_files_python.txt +++ b/tests/file_lists/lib_files_python.txt @@ -1,10 +1,12 @@ -custom_nodes -libazurestorage.so -libazurestorage.so.7->libazurestorage.so -libazurestorage.so.7.5->libazurestorage.so -libcpprest.so -libcpprest.so.2.10->libcpprest.so +libazure-core.so->libazure-core.so.1.17.0-beta.1 +libazure-core.so.1.17.0-beta.1 +libazure-storage-blobs.so->libazure-storage-blobs.so.12.18.0 +libazure-storage-blobs.so.12.18.0 +libazure-storage-common.so->libazure-storage-common.so.12.14.0 +libazure-storage-common.so.12.14.0 +libazure-storage-files-shares.so->libazure-storage-files-shares.so.12.18.0 +libazure-storage-files-shares.so.12.18.0 libespeak-ng.so->libespeak-ng.so.1 libespeak-ng.so.1->libespeak-ng.so.1.52.0.1 libespeak-ng.so.1.52.0.1 diff --git a/tests/file_lists/test_release_files.sh b/tests/file_lists/test_release_files.sh index 3cf9972385..5d1e3a2028 100755 --- a/tests/file_lists/test_release_files.sh +++ b/tests/file_lists/test_release_files.sh @@ -51,18 +51,6 @@ else fi fi -# /ovms/lib/custom_nodes -input_file="/test/lib_custom_nodes_files.txt" -test_path="/ovms/lib/custom_nodes" -output="$(diff <(cat $input_file) <(ls -l $test_path | awk '{print $9 $10 $11}'))" -if [[ -n $output ]] -then - printf -- "ERROR: $test_path against $input_file- File list mismatch: \n%s\n" "$output" - errors=$((errors+1)) -else - printf -- "SUCCESS: $test_path Files list match.\n" -fi - # check for errors if [ "$errors" -ne "0" ]; then printf -- "$errors error[s] in files list check. Please verify release image contents.\n" diff --git a/third_party/azure/BUILD b/third_party/azure/BUILD index c8e6de1692..a8b804e5e3 100644 --- a/third_party/azure/BUILD +++ b/third_party/azure/BUILD @@ -18,16 +18,64 @@ package( default_visibility = ["//visibility:public"], ) +cc_library( + name = "azure_core", + srcs = glob(["lib/libazure-core.so"]), + hdrs = glob([ + "include/azure/core/**/*.hpp", + "include/azure/core/**/*.h", + "include/azure/core.hpp", + ]), + strip_include_prefix = "include", + linkstatic = False, + data = glob(["lib/libazure-core.so.*"]), +) + +cc_library( + name = "azure_storage_common", + srcs = glob(["lib/libazure-storage-common.so"]), + hdrs = glob([ + "include/azure/storage/common/**/*.hpp", + "include/azure/storage/common/**/*.h", + ]), + strip_include_prefix = "include", + linkstatic = False, + data = glob(["lib/libazure-storage-common.so.*"]), + deps = [":azure_core"], +) + +cc_library( + name = "azure_storage_blobs", + srcs = glob(["lib/libazure-storage-blobs.so"]), + hdrs = glob([ + "include/azure/storage/blobs/**/*.hpp", + "include/azure/storage/blobs/**/*.h", + "include/azure/storage/blobs.hpp", + ]), + strip_include_prefix = "include", + linkstatic = False, + data = glob(["lib/libazure-storage-blobs.so.*"]), + deps = [":azure_storage_common"], +) + +cc_library( + name = "azure_storage_files_shares", + srcs = glob(["lib/libazure-storage-files-shares.so"]), + hdrs = glob([ + "include/azure/storage/files/**/*.hpp", + "include/azure/storage/files/**/*.h", + "include/azure/storage/files/shares.hpp", + ]), + strip_include_prefix = "include", + linkstatic = False, + data = glob(["lib/libazure-storage-files-shares.so.*"]), + deps = [":azure_storage_common"], +) + cc_library( name = "storage", - srcs = [ - "Microsoft.WindowsAzure.Storage/build.release/Binaries/libazurestorage.so.7.5", - "Microsoft.WindowsAzure.Storage/build.release/Binaries/libazurestorage.so.7", - "Microsoft.WindowsAzure.Storage/build.release/Binaries/libazurestorage.so", + deps = [ + ":azure_storage_blobs", + ":azure_storage_files_shares", ], - hdrs = glob([ - "Microsoft.WindowsAzure.Storage/includes/**/*.h", - "Microsoft.WindowsAzure.Storage/includes/**/*.dat", - ]), - strip_include_prefix = "Microsoft.WindowsAzure.Storage/includes/", )