From af8d27bd6097894656264028a34125ce36bdfabc Mon Sep 17 00:00:00 2001 From: benjobs Date: Thu, 3 Sep 2026 21:41:54 +0800 Subject: [PATCH 1/9] [Build] Establish the StreamPark 3.0 architecture baseline --- .asf.yaml | 16 +- .github/actions/workflow-telemetry-action | 1 - .github/boring-cyborg.yml | 3 - .github/workflows/e2e.yml | 252 ---- .gitignore | 1 + AGENTS.md | 88 +- NOTICE | 2 +- build.sh | 2 +- dist-material/release-docs/NOTICE | 2 +- pom.xml | 63 +- streampark | 510 ++++++++ streampark-common/pom.xml | 20 +- .../streampark/common/conf/CommonConfig.java | 182 --- .../streampark/common/conf/ConfigKeys.java | 396 ------- .../streampark/common/conf/ConfigOption.java | 159 --- .../common/conf/InternalConfigHolder.java | 154 --- .../common/conf/InternalOption.java | 75 -- .../common/conf/K8sFlinkConfig.java | 83 -- .../streampark/common/conf/Workspace.java | 254 ---- .../configuration/CommandLineParser.java | 87 ++ .../configuration/CommandLineTokenizer.java | 105 ++ .../common/configuration/ConfigException.java | 49 + .../common/configuration/ConfigOption.java | 178 +++ .../common/configuration/ConfigOptions.java | 284 +++++ .../common/configuration/ConfigOrigin.java | 94 ++ .../common/configuration/ConfigPrefix.java | 100 ++ .../common/configuration/ConfigSource.java | 45 + .../ConfigValueType.java} | 33 +- .../configuration/ConfigValueTypes.java | 259 ++++ .../common/configuration/Configuration.java | 416 +++++++ .../configuration/ConfigurationFormat.java | 60 + .../configuration/ConfigurationLoader.java | 71 ++ .../configuration/ConfigurationParser.java | 183 +++ .../Constants.java | 34 +- .../common/configuration/DataSize.java | 149 +++ .../common/configuration/FlinkOptions.java | 111 ++ .../common/configuration}/FlinkRunOption.java | 111 +- .../configuration/GlobalConfiguration.java | 82 ++ .../configuration/GroupedOptionsParser.java | 77 ++ .../configuration/JvmOptionsParser.java | 76 ++ .../common/configuration/ReadableConfig.java | 61 + .../common/configuration/SensitiveKeys.java | 76 ++ .../common/configuration/Workspace.java | 262 +++++ .../option/ApplicationOptions.java | 72 ++ .../configuration/option/CoreOptions.java | 38 +- .../option/DeploymentOptions.java | 57 + .../configuration/option/DockerOptions.java | 71 ++ .../configuration/option/HadoopOptions.java | 97 ++ .../configuration/option/MavenOptions.java | 66 ++ .../option/WorkspaceOptions.java | 51 + .../configuration/option/YarnOptions.java | 52 + .../option/package-info.java} | 30 +- .../common/configuration/package-info.java | 48 + .../common/constants/DateConstants.java | 30 - .../common/{conf => core}/FlinkVersion.java | 66 +- .../common/{conf => core}/SparkVersion.java | 10 +- ...va => FlinkKubernetesRestExposedType.java} | 10 +- .../streampark/common/enums/Semantic.java | 47 - .../common/util/ClassLoaderUtils.java | 2 +- .../streampark/common/util/ConfigUtils.java | 181 --- .../streampark/common/util/DeflaterUtils.java | 88 +- .../streampark/common/util/FileUtils.java | 2 +- .../common/util/FlinkConfigurationUtils.java | 473 +++++--- .../streampark/common/util/HBaseClient.java | 83 -- .../common/util/HadoopConfigUtils.java | 57 +- .../streampark/common/util/HadoopUtils.java | 101 +- .../streampark/common/util/JdbcUtils.java | 332 ------ .../streampark/common/util/MongoConfig.java | 189 --- .../streampark/common/util/OkHttpUtils.java | 130 ++ .../{SafePathUtils.java => PathUtils.java} | 28 +- .../common/util/PropertiesUtils.java | 246 ---- .../streampark/common/util/RedisClient.java | 143 --- .../streampark/common/util/RedisEndpoint.java | 377 ------ .../streampark/common/util/RedisUtils.java | 479 -------- .../common/util/SqlConvertUtils.java | 264 ----- .../common/util/StringCastUtils.java | 58 - .../apache/streampark/common/util/Tuple.java | 64 + .../apache/streampark/common/util/Tuple1.java | 119 ++ .../apache/streampark/common/util/Tuple2.java | 164 +++ .../apache/streampark/common/util/Tuple3.java | 172 +++ .../apache/streampark/common/util/Tuple4.java | 188 +++ .../apache/streampark/common/util/Utils.java | 6 +- .../streampark/common/util/YamlParser.java | 489 ++++++++ .../streampark/common/util/YarnUtils.java | 35 +- .../common/util/ZooKeeperUtils.java | 165 --- .../configuration/CommandLineParserTest.java | 85 ++ .../ConfigurationParserTest.java | 92 ++ .../configuration/ConfigurationTest.java | 261 +++++ .../common/configuration/WorkspaceTest.java | 136 +++ .../common/core/FlinkVersionTest.java | 47 + .../common/util/DeflaterUtilsTest.java | 70 ++ .../util/FlinkConfigurationUtilsTest.java | 179 +++ .../common/util/PropertiesUtilsTestCase.java | 75 -- .../common/util/YamlParserTest.java | 100 ++ .../streampark-console-service/pom.xml | 23 +- .../src/main/assembly/bin/streampark.sh | 2 +- .../console/base/config/ConsoleOptions.java | 44 + .../SpringConfigurationInitializer.java | 158 +++ .../console/base/config/SpringProperties.java | 57 +- .../console/base/config/package-info.java | 14 +- .../console/base/util/BashJavaUtils.java | 14 +- .../console/base/util/WebUtils.java | 152 ++- .../core/assembler/FlinkEnvAssembler.java | 10 + .../console/core/bean/Dependency.java | 6 +- .../console/core/bean/MavenConfig.java | 30 +- .../core/controller/FlinkEnvController.java | 3 +- .../core/controller/ProxyController.java | 34 +- .../console/core/entity/FlinkApplication.java | 40 +- .../core/entity/FlinkApplicationBackup.java | 6 +- .../core/entity/FlinkApplicationConfig.java | 89 +- .../console/core/entity/FlinkCluster.java | 14 +- .../console/core/entity/FlinkEnv.java | 136 +-- .../console/core/entity/FlinkSql.java | 41 +- .../console/core/entity/Project.java | 13 +- .../console/core/entity/SparkApplication.java | 33 +- .../core/entity/SparkApplicationBackup.java | 6 +- .../console/core/entity/SparkEnv.java | 2 +- .../console/core/entity/Variable.java | 2 +- .../console/core/runner/EnvInitializer.java | 128 +- .../console/core/runner/QuickStartRunner.java | 4 +- .../console/core/service/ProxyService.java | 13 +- .../FlinkApplicationActionServiceImpl.java | 94 +- .../FlinkApplicationBackupServiceImpl.java | 13 +- ...nkApplicationBuildPipelineServiceImpl.java | 32 +- .../FlinkApplicationConfigServiceImpl.java | 2 +- .../impl/FlinkApplicationInfoServiceImpl.java | 8 +- .../FlinkApplicationManageServiceImpl.java | 32 +- .../SparkApplicationActionServiceImpl.java | 17 +- .../SparkApplicationBackupServiceImpl.java | 9 +- ...rkApplicationBuildPipelineServiceImpl.java | 10 +- .../impl/SparkApplicationInfoServiceImpl.java | 4 +- .../SparkApplicationManageServiceImpl.java | 11 +- .../service/impl/FlinkClusterServiceImpl.java | 35 +- .../service/impl/FlinkEnvServiceImpl.java | 10 +- .../impl/FlinkSavepointServiceImpl.java | 95 +- .../service/impl/FlinkSqlServiceImpl.java | 19 +- .../core/service/impl/ProjectServiceImpl.java | 14 +- .../core/service/impl/ProxyServiceImpl.java | 241 ++-- .../service/impl/ResourceServiceImpl.java | 12 +- .../console/core/task/ProjectBuildTask.java | 2 +- .../util/ApplicationBuildPipelineUtils.java | 6 +- .../ApplicationEntitySupport.java | 11 +- .../core/util/ApplicationEntityUtils.java | 11 +- .../core/util/DataTypeConverterUtils.java | 196 ---- .../util/FlinkApplicationConfigUtils.java | 116 ++ .../console/core/util/FlinkEnvUtils.java | 173 +++ .../core/util/FlinkShellCommandBuilder.java | 256 ++++ .../core/util/YarnQueueLabelExpression.java | 7 +- .../core/watcher/FlinkClusterWatcher.java | 7 +- .../watcher/FlinkK8sChangeEventListener.java | 4 +- .../core/watcher/FlinkK8sWatcherWrapper.java | 20 +- .../console/system/entity/User.java | 2 +- .../system/runner/StartedUpRunner.java | 2 +- .../console/SpringIntegrationTestBase.java | 10 +- .../console/SpringUnitTestBase.java | 15 +- .../SpringConfigurationInitializerTest.java | 97 ++ .../base/util/DependencyUtilsTest.java | 6 +- .../console/base/util/ShaHashUtilsTest.java | 2 +- .../console/base/util/WebUtilsTest.java | 105 ++ .../FlinkApplicationAssemblerTest.java | 2 + .../console/core/entity/FlinkSqlTest.java | 83 ++ .../service/FlinkSavepointServiceTest.java | 58 +- .../console/core/util/FlinkEnvUtilsTest.java | 101 ++ .../util/FlinkShellCommandBuilderTest.java | 25 +- .../streampark-console-webapp/.env | 2 +- .../.env.development | 4 +- .../streampark-console-webapp/package.json | 2 +- .../src/views/flink/app/EditStreamPark.vue | 22 +- streampark-e2e/README.md | 150 --- streampark-e2e/pom.xml | 141 --- streampark-e2e/streampark-e2e-case/pom.xml | 38 - .../streampark/e2e/cases/AlarmTest.java | 149 --- .../streampark/e2e/cases/EnvironmentTest.java | 191 --- .../e2e/cases/ExternalLinkTest.java | 123 -- .../Flink116OnRemoteClusterDeployTest.java | 104 -- .../Flink116OnYarnClusterDeployTest.java | 151 --- .../Flink117OnRemoteClusterDeployTest.java | 104 -- .../Flink117OnYarnClusterDeployTest.java | 152 --- .../Flink118OnRemoteClusterDeployTest.java | 104 -- .../Flink118OnYarnClusterDeployTest.java | 152 --- .../Flink120OnRemoteClusterDeployTest.java | 104 -- .../Flink120OnYarnClusterDeployTest.java | 152 --- .../e2e/cases/FlinkSQL116OnYarnTest.java | 330 ------ .../e2e/cases/FlinkSQL117OnYarnTest.java | 357 ------ .../e2e/cases/FlinkSQL118OnYarnTest.java | 371 ------ .../e2e/cases/FlinkSQL120OnYarnTest.java | 269 ----- .../e2e/cases/MemberManagementTest.java | 117 -- .../e2e/cases/ProjectsManagementTest.java | 126 -- .../e2e/cases/RoleManagementTest.java | 113 -- .../e2e/cases/TeamManagementTest.java | 114 -- .../e2e/cases/TokenManagementTest.java | 116 -- .../e2e/cases/UploadManagementTest.java | 141 --- .../e2e/cases/UserManagementTest.java | 106 -- .../e2e/cases/VariableManagementTest.java | 117 -- .../streampark/e2e/cases/YarnQueueTest.java | 113 -- .../streampark/e2e/pages/LoginPage.java | 91 -- .../e2e/pages/common/CommonFactory.java | 95 -- .../e2e/pages/common/Constants.java | 60 - .../e2e/pages/common/NavBarPage.java | 106 -- .../e2e/pages/flink/ApacheFlinkPage.java | 76 -- .../e2e/pages/flink/FlinkHomePage.java | 101 -- .../flink/applications/ApplicationForm.java | 280 ----- .../flink/applications/ApplicationsPage.java | 188 --- .../flink/applications/FlinkSQLEditor.java | 110 -- .../flink/clusters/ClusterDetailForm.java | 88 -- .../e2e/pages/flink/clusters/CommonForm.java | 86 -- .../flink/clusters/FlinkClustersPage.java | 144 --- .../e2e/pages/flink/clusters/RemoteForm.java | 44 - .../pages/flink/clusters/YarnSessionForm.java | 71 -- .../e2e/pages/resource/ProjectsPage.java | 243 ---- .../e2e/pages/resource/ResourcePage.java | 79 -- .../e2e/pages/resource/UploadsPage.java | 206 ---- .../e2e/pages/resource/VariablesPage.java | 157 --- .../e2e/pages/setting/ExternalLinkPage.java | 153 --- .../e2e/pages/setting/SettingPage.java | 85 -- .../e2e/pages/setting/YarnQueuePage.java | 137 --- .../e2e/pages/setting/alarm/AlarmPage.java | 135 --- .../setting/alarm/AlertTypeDetailForm.java | 102 -- .../e2e/pages/setting/alarm/CommonForm.java | 64 - .../setting/alarm/DingTalkAlertForm.java | 84 -- .../pages/setting/alarm/EmailAlertForm.java | 44 - .../pages/setting/alarm/LarkAlertForm.java | 68 -- .../e2e/pages/setting/alarm/SMSAlertForm.java | 32 - .../e2e/pages/setting/env/CommonForm.java | 36 - .../pages/setting/env/DockerSettingForm.java | 85 -- .../pages/setting/env/EmailSettingForm.java | 100 -- .../setting/env/EnvironmentDetailForm.java | 68 -- .../pages/setting/env/EnvironmentPage.java | 91 -- .../pages/setting/env/IngressSettingForm.java | 54 - .../pages/setting/env/MavenSettingForm.java | 103 -- .../pages/system/MemberManagementPage.java | 198 ---- .../e2e/pages/system/RoleManagementPage.java | 162 --- .../e2e/pages/system/SystemPage.java | 100 -- .../e2e/pages/system/TeamManagementPage.java | 139 --- .../e2e/pages/system/TokenManagementPage.java | 130 -- .../e2e/pages/system/UserManagementPage.java | 183 --- .../test/resources/docker/basic/Dockerfile | 18 - .../docker/basic/docker-compose.yaml | 46 - .../src/test/resources/docker/basic/gitconfig | 19 - .../resources/docker/environment/Dockerfile | 18 - .../docker/environment/docker-compose.yaml | 60 - .../docker/flink-1.16-on-remote/Dockerfile | 18 - .../flink-1.16-on-remote/docker-compose.yaml | 90 -- .../docker/flink-1.16-on-yarn/Dockerfile | 35 - .../flink-1.16-on-yarn/docker-compose.config | 44 - .../flink-1.16-on-yarn/docker-compose.yaml | 163 --- .../docker/flink-1.17-on-remote/Dockerfile | 18 - .../flink-1.17-on-remote/docker-compose.yaml | 90 -- .../docker/flink-1.17-on-yarn/Dockerfile | 35 - .../flink-1.17-on-yarn/docker-compose.config | 44 - .../flink-1.17-on-yarn/docker-compose.yaml | 163 --- .../docker/flink-1.18-on-remote/Dockerfile | 18 - .../flink-1.18-on-remote/docker-compose.yaml | 90 -- .../docker/flink-1.18-on-yarn/Dockerfile | 35 - .../flink-1.18-on-yarn/docker-compose.config | 44 - .../flink-1.18-on-yarn/docker-compose.yaml | 163 --- .../docker/flink-1.20-on-remote/Dockerfile | 18 - .../flink-1.20-on-remote/docker-compose.yaml | 90 -- .../docker/flink-1.20-on-yarn/Dockerfile | 35 - .../flink-1.20-on-yarn/docker-compose.config | 44 - .../flink-1.20-on-yarn/docker-compose.yaml | 163 --- streampark-e2e/streampark-e2e-core/pom.xml | 30 - .../apache/streampark/e2e/core/Constants.java | 37 - .../streampark/e2e/core/StreamPark.java | 40 - .../e2e/core/StreamParkExtension.java | 241 ---- .../streampark/e2e/core/TestDescription.java | 51 - .../src/main/resources/docker-java.properties | 20 - .../src/main/resources/log4j2.xml | 31 - streampark-flink/pom.xml | 2 - .../streampark-flink-client-api/pom.xml | 31 - .../streampark/flink/client/FlinkClient.java | 96 +- .../flink/client/bean/DeployRequestTrait.java | 41 - .../flink/client/bean/HdfsWorkspace.java | 71 -- .../client/bean/SavepointCancelOptions.java | 58 - .../client/bean/SavepointRequestTrait.java | 52 - .../client/bean/SavepointTriggerOptions.java | 42 - .../flink/client/bean/SubmitRequest.java | 469 -------- .../{bean => request}/CancelRequest.java | 30 +- .../client/request/ClientRequestUtils.java | 53 + .../ClusterRequest.java} | 42 +- .../{bean => request}/DeployRequest.java | 19 +- .../{bean => request}/JobClientTarget.java | 4 +- .../KubernetesDeploySpec.java} | 41 +- .../SavepointRequest.java} | 44 +- .../ShutdownRequest.java} | 13 +- .../SubmitApplicationSpec.java | 16 +- .../{bean => request}/SubmitClusterSpec.java | 10 +- .../flink/client/request/SubmitRequest.java | 150 +++ .../TriggerSavepointRequest.java | 19 +- .../flink/client/request/package-info.java} | 15 +- .../{bean => response}/CancelResponse.java | 5 +- .../{bean => response}/DeployResponse.java | 5 +- .../{bean => response}/SavepointResponse.java | 6 +- .../ShutdownResponse.java} | 7 +- .../{bean => response}/SubmitResponse.java | 10 +- .../flink/client/response/package-info.java | 7 +- .../flink/proxy/FlinkShimsProxy.java | 106 +- .../flink/client/FlinkClientTest.java} | 10 +- .../ClientRequestUtilsTest.java} | 18 +- .../flink/proxy/FlinkShimsProxyTest.java | 94 ++ .../flink/client/FlinkClientEntrypoint.java | 46 +- .../client/bean/FlinkJobGraphBuilder.java | 192 +++ .../flink/client/bean/RemoteWorkspace.java} | 76 +- .../client/bean/SessionClusterRestClient.java | 247 ++++ .../bean/SubmissionConfigurationBuilder.java | 454 +++++++ .../client/bean/SubmitRequestResolver.java | 297 +++++ .../FlinkConfigurationOps.java | 12 +- .../configuration}/FlinkSavepointOptions.java | 6 +- .../client/impl/AbstractFlinkClient.java | 409 +++++++ .../AbstractKubernetesNativeClient.java} | 83 +- .../AbstractYarnClient.java} | 100 +- .../KubernetesNativeApplicationClient.java | 78 +- .../impl/KubernetesNativeSessionClient.java | 168 ++- .../flink/client/impl/LocalClient.java | 78 +- .../flink/client/impl/RemoteClient.java | 113 +- .../client/impl/YarnApplicationClient.java | 117 +- .../flink/client/impl/YarnPerJobClient.java | 118 +- .../flink/client/impl/YarnSessionClient.java | 151 +-- .../client/tool/FlinkSessionSubmitHelper.java | 243 ---- .../flink/client/trait/FlinkClientTrait.java | 1042 ----------------- ...st.java => SubmitRequestResolverTest.java} | 73 +- .../flink/client/test/YarnPerJobTestCase.java | 4 +- .../streampark-flink-kubernetes/pom.xml | 2 +- ...ava => DefaultFlinkKubernetesWatcher.java} | 14 +- ...va => FlinkKubernetesWatchController.java} | 6 +- ...tcher.java => FlinkKubernetesWatcher.java} | 2 +- ...ava => FlinkKubernetesWatcherFactory.java} | 14 +- .../flink/kubernetes/FlinkTrackConfig.java | 33 +- ...va => KubernetesDeploymentEventCache.java} | 18 +- ...s.java => KubernetesPodTemplateFiles.java} | 4 +- .../flink/kubernetes/KubernetesRetriever.java | 4 +- ...a => LazyStartFlinkKubernetesWatcher.java} | 6 +- .../flink/kubernetes/PodTemplateTool.java | 20 +- .../configuration/KubernetesOptions.java | 87 ++ .../configuration/package-info.java | 19 + ...de.java => FlinkKubernetesDeployMode.java} | 12 +- .../helper/KubernetesWatcherHelper.java | 22 +- .../kubernetes/ingress/IngressStrategy.java | 10 +- .../flink/kubernetes/model/ClusterKey.java | 6 +- ....java => KubernetesDeploymentEventCV.java} | 2 +- ...sEventKey.java => KubernetesEventKey.java} | 6 +- ...lates.java => KubernetesPodTemplates.java} | 14 +- .../flink/kubernetes/model/TrackId.java | 10 +- .../watcher/FlinkCheckpointWatcher.java | 6 +- .../watcher/FlinkJobStatusWatcher.java | 12 +- ....java => FlinkKubernetesEventWatcher.java} | 16 +- .../watcher/FlinkMetricWatcher.java | 6 +- ....java => SparkKubernetesPodTemplates.java} | 14 +- .../flink/packer/PackerResourceGC.java | 6 +- .../flink/packer/docker/DockerRetriever.java | 40 +- .../docker/FlinkDockerfileTemplateTrait.java | 2 +- .../docker/SparkDockerfileTemplateTrait.java | 2 +- .../flink/packer/maven/MavenTool.java | 21 +- .../packer/pipeline/FlinkBuildParam.java | 8 +- .../FlinkK8sApplicationBuildRequest.java | 10 +- .../pipeline/FlinkK8sSessionBuildRequest.java | 2 +- .../FlinkRemotePerJobBuildRequest.java | 2 +- .../SparkK8sApplicationBuildRequest.java | 10 +- .../packer/pipeline/YarnJarUploader.java | 4 +- .../flink/packer/DockerClientTest.java | 13 +- .../flink/packer/MavenToolTest.java | 4 +- .../streampark-flink-proxy/pom.xml | 49 - .../flink/proxy/FlinkShimsProxyTest.java | 72 -- .../streampark-flink-shims/pom.xml | 23 +- .../streampark-flink-shims-base-v2/pom.xml | 109 -- .../flink/core/FlinkSqlExecutor.java | 293 ----- .../flink/core/FlinkStreamTableTraitV2.java | 691 ----------- .../core/FlinkStreamingInitializerV2.java | 193 --- .../flink/core/FlinkTableInitializerV2.java | 297 ----- .../streampark/flink/core/SqlCommand.java | 180 --- .../flink/core/TableEnvConfigFunction.java | 34 - .../flink/core/conf/FlinkConfiguration.java | 40 - .../streampark/flink/util/FlinkUtils.java | 79 -- .../org/apache/streampark/RegExpTest.java | 92 -- .../streampark-flink-shims-base/pom.xml | 4 + .../configuration/FlinkJobParameters.java | 130 ++ .../FlinkRuntimeConfiguration.java | 105 ++ .../flink/configuration/package-info.java | 33 + ...entTrait.java => AbstractFlinkClient.java} | 4 +- ...ava => AbstractFlinkKubernetesClient.java} | 4 +- .../flink/core/AbstractFlinkStreamTable.java | 228 ++++ .../flink/core/AbstractFlinkTable.java} | 218 ++-- .../flink/core/FlinkParameterUtils.java | 87 -- .../flink/core/FlinkSqlExecutor.java | 95 +- .../flink/core/FlinkStreamInitializer.java | 162 +++ .../flink/core/FlinkStreamTableTrait.java | 845 ------------- .../flink/core/FlinkStreamingInitializer.java | 193 --- .../flink/core/FlinkTableInitializer.java | 323 +++-- .../flink/core/FlinkTableTrait.java | 384 ------ .../streampark/flink/core/SqlCommand.java | 27 +- .../flink/core/SqlCommandParser.java | 34 +- .../streampark/flink/core/SqlSplitter.java | 9 +- .../flink/core/StreamEnvConfig.java | 30 - .../flink/core/StreamTableEnvConfig.java | 35 - .../streampark/flink/core/TableEnvConfig.java | 30 - .../flink/core/TableEnvConfigFunction.java | 34 - .../flink/core/bean/StreamContextSpec.java} | 23 +- .../StreamTableContextSpec.java} | 29 +- .../flink/core/bean/TableContextSpec.java} | 24 +- .../flink/core/conf/FlinkConfiguration.java | 40 - .../flink/core/conf/ParameterCli.java | 230 ---- .../flink/util}/FlinkParameterUtils.java | 22 +- .../streampark/flink/util/FlinkUtils.java | 6 +- .../FlinkJobParametersTest.java} | 34 +- .../flink/core/FlinkSqlExecutorTest.java} | 27 +- .../flink/core/SqlCommandParserTest.java} | 32 +- .../streampark-flink-shims-test/pom.xml | 100 -- .../flink/core/test/FetchMillisecond.java | 49 - .../flink/core/test/FlinkSqlExecuteTest.java | 77 -- .../core/test/FlinkSqlValidationTest.java | 236 ---- .../streampark-flink-shims_flink-1.17/pom.xml | 114 -- .../flink/core/FlinkClusterClient.java | 63 - .../flink/core/StreamTableContext.java | 47 - .../streampark/flink/core/TableExt.java | 66 -- .../streampark-flink-shims_flink-1.18/pom.xml | 4 - .../flink/core/FlinkClusterClient.java | 2 +- .../flink/core/FlinkKubernetesClient.java | 2 +- .../flink/core/StreamTableContext.java | 48 +- .../streampark/flink/core/TableContext.java | 18 +- .../streampark/flink/core/TableExt.java | 66 -- .../streampark-flink-shims_flink-1.19/pom.xml | 3 - .../flink/core/FlinkClusterClient.java | 14 +- .../flink/core/FlinkKubernetesClient.java | 2 +- .../flink/core/StreamTableContext.java | 49 +- .../streampark/flink/core/TableContext.java | 15 +- .../streampark/flink/core/TableExt.java | 66 -- .../streampark-flink-shims_flink-1.20/pom.xml | 3 - .../flink/core/FlinkClusterClient.java | 2 +- .../flink/core/FlinkKubernetesClient.java | 2 +- .../flink/core/StreamTableContext.java | 49 +- .../streampark/flink/core/TableContext.java | 15 +- .../streampark/flink/core/TableExt.java | 66 -- .../streampark-flink-shims_flink-2.0/pom.xml | 2 +- .../flink/core/FlinkClusterClient.java | 13 +- .../flink/core/FlinkKubernetesClient.java | 2 +- .../flink/core/StreamTableContext.java | 245 +--- .../streampark/flink/core/TableContext.java | 119 +- .../streampark/flink/core/TableExt.java | 62 - .../streampark-flink-shims_flink-2.1/pom.xml | 2 +- .../flink/core/FlinkClusterClient.java | 2 +- .../flink/core/FlinkKubernetesClient.java | 2 +- .../flink/core/StreamTableContext.java | 288 +---- .../streampark/flink/core/TableContext.java | 164 +-- .../streampark/flink/core/TableExt.java | 62 - .../streampark-flink-shims_flink-2.2/pom.xml | 2 +- .../flink/core/FlinkClusterClient.java | 2 +- .../flink/core/FlinkKubernetesClient.java | 2 +- .../flink/core/StreamTableContext.java | 317 +---- .../streampark/flink/core/TableContext.java | 187 +-- .../streampark/flink/core/TableExt.java | 62 - .../streampark-flink-shims_flink-2.3/pom.xml | 2 +- .../flink/core/FlinkClusterClient.java | 2 +- .../flink/core/FlinkKubernetesClient.java | 2 +- .../flink/core/StreamTableContext.java | 325 +---- .../streampark/flink/core/TableContext.java | 187 +-- .../streampark/flink/core/TableExt.java | 62 - .../streampark/flink/cli/SqlClient.java | 87 +- .../streampark/flink/cli/SqlClientTest.java | 42 +- streampark-flink/streampark-flink-udf/pom.xml | 69 -- .../streampark/flink/udf/DateFormat.java | 40 - .../pom.xml | 22 +- .../streampark/common/util/Implicits.scala | 0 .../streampark/common/util/Logger.scala | 0 streampark-spark/pom.xml | 46 +- .../streampark-spark-cli/create_app.sh | 412 ------- .../streampark-spark-cli/create_conf.sh | 193 --- .../streampark-spark-cli/default.properties | 147 --- streampark-spark/streampark-spark-cli/pom.xml | 30 - .../spark/configuration/SparkOptions.java | 96 ++ .../spark/configuration/package-info.java | 7 +- .../streampark/spark/client/SparkClient.scala | 2 +- .../spark/client/bean/CancelRequest.scala | 2 +- .../spark/client/bean/SubmitRequest.scala | 35 +- .../spark/client/proxy/SparkShimsProxy.scala | 9 +- .../spark/client/impl/YarnClient.scala | 15 +- .../configuration/SparkRuntimeOptions.java | 46 + .../spark/configuration/package-info.java | 19 + .../spark/core/util/ParameterTool.java | 81 -- .../spark/core/util/SQLContextUtil.scala | 52 - .../spark/core/util/SparkSqlExecutor.scala | 34 +- .../spark/core/util/SqlCommandParser.scala | 8 +- 481 files changed, 13867 insertions(+), 28299 deletions(-) delete mode 160000 .github/actions/workflow-telemetry-action delete mode 100644 .github/workflows/e2e.yml create mode 100755 streampark delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/conf/CommonConfig.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/conf/ConfigKeys.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/conf/ConfigOption.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/conf/InternalConfigHolder.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/conf/InternalOption.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/conf/K8sFlinkConfig.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/conf/Workspace.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineParser.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineTokenizer.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigException.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOption.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOptions.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOrigin.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigPrefix.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigSource.java rename streampark-common/src/main/java/org/apache/streampark/common/{enums/ApiType.java => configuration/ConfigValueType.java} (51%) create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigValueTypes.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationFormat.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationLoader.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationParser.java rename streampark-common/src/main/java/org/apache/streampark/common/{constants => configuration}/Constants.java (67%) create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/DataSize.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/FlinkOptions.java rename {streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf => streampark-common/src/main/java/org/apache/streampark/common/configuration}/FlinkRunOption.java (71%) create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/GlobalConfiguration.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/GroupedOptionsParser.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/JvmOptionsParser.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/ReadableConfig.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/SensitiveKeys.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/Workspace.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/option/ApplicationOptions.java rename streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/WeChatAlertForm.java => streampark-common/src/main/java/org/apache/streampark/common/configuration/option/CoreOptions.java (51%) create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/option/DeploymentOptions.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/option/DockerOptions.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/option/HadoopOptions.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/option/MavenOptions.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/option/WorkspaceOptions.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/option/YarnOptions.java rename streampark-common/src/main/java/org/apache/streampark/common/{enums/TableMode.java => configuration/option/package-info.java} (61%) create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/configuration/package-info.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/constants/DateConstants.java rename streampark-common/src/main/java/org/apache/streampark/common/{conf => core}/FlinkVersion.java (85%) rename streampark-common/src/main/java/org/apache/streampark/common/{conf => core}/SparkVersion.java (95%) rename streampark-common/src/main/java/org/apache/streampark/common/enums/{FlinkK8sRestExposedType.java => FlinkKubernetesRestExposedType.java} (83%) delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/enums/Semantic.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/ConfigUtils.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/HBaseClient.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/MongoConfig.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/OkHttpUtils.java rename streampark-common/src/main/java/org/apache/streampark/common/util/{SafePathUtils.java => PathUtils.java} (78%) delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/PropertiesUtils.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/RedisClient.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/RedisEndpoint.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/RedisUtils.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/SqlConvertUtils.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/StringCastUtils.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/Tuple.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/Tuple1.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/Tuple2.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/Tuple3.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/Tuple4.java create mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java delete mode 100644 streampark-common/src/main/java/org/apache/streampark/common/util/ZooKeeperUtils.java create mode 100644 streampark-common/src/test/java/org/apache/streampark/common/configuration/CommandLineParserTest.java create mode 100644 streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationParserTest.java create mode 100644 streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationTest.java create mode 100644 streampark-common/src/test/java/org/apache/streampark/common/configuration/WorkspaceTest.java create mode 100644 streampark-common/src/test/java/org/apache/streampark/common/core/FlinkVersionTest.java create mode 100644 streampark-common/src/test/java/org/apache/streampark/common/util/DeflaterUtilsTest.java create mode 100644 streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java delete mode 100644 streampark-common/src/test/java/org/apache/streampark/common/util/PropertiesUtilsTestCase.java create mode 100644 streampark-common/src/test/java/org/apache/streampark/common/util/YamlParserTest.java create mode 100644 streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/ConsoleOptions.java create mode 100644 streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringConfigurationInitializer.java rename streampark-common/src/main/java/org/apache/streampark/common/enums/CatalogType.java => streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/package-info.java (82%) rename streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/{entity => util}/ApplicationEntitySupport.java (86%) delete mode 100644 streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/DataTypeConverterUtils.java create mode 100644 streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkApplicationConfigUtils.java create mode 100644 streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkEnvUtils.java create mode 100644 streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilder.java create mode 100644 streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/config/SpringConfigurationInitializerTest.java create mode 100644 streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/WebUtilsTest.java create mode 100644 streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/entity/FlinkSqlTest.java create mode 100644 streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkEnvUtilsTest.java rename streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/conf/ParameterCliTest.java => streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilderTest.java (69%) delete mode 100644 streampark-e2e/README.md delete mode 100644 streampark-e2e/pom.xml delete mode 100644 streampark-e2e/streampark-e2e-case/pom.xml delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/AlarmTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/EnvironmentTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ExternalLinkTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnRemoteClusterDeployTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnYarnClusterDeployTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnRemoteClusterDeployTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnYarnClusterDeployTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnRemoteClusterDeployTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnYarnClusterDeployTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnRemoteClusterDeployTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnYarnClusterDeployTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL116OnYarnTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL117OnYarnTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL118OnYarnTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL120OnYarnTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/MemberManagementTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/RoleManagementTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TeamManagementTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TokenManagementTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UploadManagementTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UserManagementTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/VariableManagementTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/YarnQueueTest.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/LoginPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/Constants.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/NavBarPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/ApacheFlinkPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/FlinkHomePage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationsPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/FlinkSQLEditor.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/ClusterDetailForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/CommonForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/FlinkClustersPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/RemoteForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/YarnSessionForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ResourcePage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/UploadsPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/VariablesPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/ExternalLinkPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/SettingPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/YarnQueuePage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/AlarmPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/AlertTypeDetailForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/CommonForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/DingTalkAlertForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/EmailAlertForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/LarkAlertForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/SMSAlertForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/CommonForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/DockerSettingForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EmailSettingForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EnvironmentDetailForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EnvironmentPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/IngressSettingForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/MavenSettingForm.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/MemberManagementPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/RoleManagementPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/SystemPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/TeamManagementPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/TokenManagementPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/UserManagementPage.java delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/Dockerfile delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/gitconfig delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/environment/Dockerfile delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/environment/docker-compose.yaml delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/Dockerfile delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/docker-compose.yaml delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/Dockerfile delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.config delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.yaml delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-remote/Dockerfile delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-remote/docker-compose.yaml delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/Dockerfile delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/docker-compose.config delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/docker-compose.yaml delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-remote/Dockerfile delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-remote/docker-compose.yaml delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/Dockerfile delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/docker-compose.config delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/docker-compose.yaml delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-remote/Dockerfile delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-remote/docker-compose.yaml delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/Dockerfile delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/docker-compose.config delete mode 100644 streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/docker-compose.yaml delete mode 100644 streampark-e2e/streampark-e2e-core/pom.xml delete mode 100644 streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/Constants.java delete mode 100644 streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamPark.java delete mode 100644 streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java delete mode 100644 streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/TestDescription.java delete mode 100644 streampark-e2e/streampark-e2e-core/src/main/resources/docker-java.properties delete mode 100644 streampark-e2e/streampark-e2e-core/src/main/resources/log4j2.xml delete mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequestTrait.java delete mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/HdfsWorkspace.java delete mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointCancelOptions.java delete mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointRequestTrait.java delete mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointTriggerOptions.java delete mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequest.java rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean => request}/CancelRequest.java (63%) create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClientRequestUtils.java rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean/AbstractDeployClientRequest.java => request/ClusterRequest.java} (62%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean => request}/DeployRequest.java (72%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean => request}/JobClientTarget.java (93%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean/KubernetesDeployParam.java => request/KubernetesDeploySpec.java} (60%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean/AbstractSavepointClientRequest.java => request/SavepointRequest.java} (63%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean/ShutDownRequest.java => request/ShutdownRequest.java} (78%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean => request}/SubmitApplicationSpec.java (88%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean => request}/SubmitClusterSpec.java (84%) create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitRequest.java rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean => request}/TriggerSavepointRequest.java (74%) rename streampark-flink/{streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/Length.java => streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/package-info.java} (74%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean => response}/CancelResponse.java (87%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean => response}/DeployResponse.java (91%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean => response}/SavepointResponse.java (86%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean/ShutDownResponse.java => response/ShutdownResponse.java} (82%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/{bean => response}/SubmitResponse.java (87%) rename streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/entity/UserManagementStatus.java => streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/package-info.java (87%) rename streampark-flink/{streampark-flink-proxy => streampark-flink-client/streampark-flink-client-api}/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java (75%) rename streampark-flink/streampark-flink-client/{streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/ExitSecurityManagerTest.java => streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/FlinkClientTest.java} (82%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/{bean/ClientBeanUtilsTest.java => request/ClientRequestUtilsTest.java} (71%) create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/FlinkJobGraphBuilder.java rename streampark-flink/streampark-flink-client/{streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ClientBeanUtils.java => streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/RemoteWorkspace.java} (52%) create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SessionClusterRestClient.java create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmissionConfigurationBuilder.java create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequestResolver.java rename streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/{impl => configuration}/FlinkConfigurationOps.java (68%) rename streampark-flink/streampark-flink-client/{streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/conf => streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration}/FlinkSavepointOptions.java (93%) create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java rename streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/{trait/KubernetesNativeClientTrait.java => impl/AbstractKubernetesNativeClient.java} (74%) rename streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/{trait/YarnClientTrait.java => impl/AbstractYarnClient.java} (70%) delete mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/tool/FlinkSessionSubmitHelper.java delete mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/FlinkClientTrait.java rename streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/{SubmitRequestTest.java => SubmitRequestResolverTest.java} (64%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/{DefaultFlinkK8sWatcher.java => DefaultFlinkKubernetesWatcher.java} (93%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/{FlinkK8sWatchController.java => FlinkKubernetesWatchController.java} (93%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/{FlinkK8sWatcher.java => FlinkKubernetesWatcher.java} (98%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/{FlinkK8sWatcherFactory.java => FlinkKubernetesWatcherFactory.java} (74%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/{K8sDeploymentEventCache.java => KubernetesDeploymentEventCache.java} (65%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/{K8sPodTemplateFiles.java => KubernetesPodTemplateFiles.java} (93%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/{LazyStartFlinkK8sWatcher.java => LazyStartFlinkKubernetesWatcher.java} (94%) create mode 100644 streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/configuration/KubernetesOptions.java create mode 100644 streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/configuration/package-info.java rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/enums/{FlinkK8sDeployMode.java => FlinkKubernetesDeployMode.java} (85%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/{K8sDeploymentEventCV.java => KubernetesDeploymentEventCV.java} (97%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/{K8sEventKey.java => KubernetesEventKey.java} (91%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/{K8sPodTemplates.java => KubernetesPodTemplates.java} (86%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/{FlinkK8sEventWatcher.java => FlinkKubernetesEventWatcher.java} (85%) rename streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/spark/kubernetes/model/{SparkK8sPodTemplates.java => SparkKubernetesPodTemplates.java} (84%) delete mode 100644 streampark-flink/streampark-flink-proxy/pom.xml delete mode 100644 streampark-flink/streampark-flink-proxy/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/pom.xml delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTraitV2.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializerV2.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializerV2.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/SqlCommand.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/TableEnvConfigFunction.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/conf/FlinkConfiguration.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/util/FlinkUtils.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/test/java/org/apache/streampark/RegExpTest.java create mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/FlinkJobParameters.java create mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/FlinkRuntimeConfiguration.java create mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/package-info.java rename streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/{FlinkClientTrait.java => AbstractFlinkClient.java} (96%) rename streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/{FlinkKubernetesClientTrait.java => AbstractFlinkKubernetesClient.java} (92%) create mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkStreamTable.java rename streampark-flink/streampark-flink-shims/{streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java => streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkTable.java} (81%) delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkParameterUtils.java create mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamInitializer.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTrait.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializer.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/StreamEnvConfig.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/StreamTableEnvConfig.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/TableEnvConfig.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/TableEnvConfigFunction.java rename streampark-flink/streampark-flink-shims/{streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/StreamEnvConfigFunction.java => streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/bean/StreamContextSpec.java} (65%) rename streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/{StreamEnvConfigFunction.java => bean/StreamTableContextSpec.java} (53%) rename streampark-flink/{streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/HashCode.java => streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/bean/TableContextSpec.java} (64%) delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/FlinkConfiguration.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/ParameterCli.java rename streampark-flink/streampark-flink-shims/{streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core => streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/util}/FlinkParameterUtils.java (76%) rename streampark-flink/{streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/ExitSecurityManager.java => streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/configuration/FlinkJobParametersTest.java} (51%) rename streampark-flink/streampark-flink-shims/{streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableContext.java => streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/FlinkSqlExecutorTest.java} (57%) rename streampark-flink/streampark-flink-shims/{streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java => streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/SqlCommandParserTest.java} (50%) delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-test/pom.xml delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FetchMillisecond.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlExecuteTest.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlValidationTest.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/pom.xml delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableExt.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableExt.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableExt.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableExt.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/TableExt.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/TableExt.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/TableExt.java delete mode 100644 streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/TableExt.java delete mode 100644 streampark-flink/streampark-flink-udf/pom.xml delete mode 100644 streampark-flink/streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/DateFormat.java rename {streampark-common-scala-bridge => streampark-scala}/pom.xml (72%) rename {streampark-common-scala-bridge => streampark-scala}/src/main/scala/org/apache/streampark/common/util/Implicits.scala (100%) rename {streampark-common-scala-bridge => streampark-scala}/src/main/scala/org/apache/streampark/common/util/Logger.scala (100%) delete mode 100644 streampark-spark/streampark-spark-cli/create_app.sh delete mode 100644 streampark-spark/streampark-spark-cli/create_conf.sh delete mode 100644 streampark-spark/streampark-spark-cli/default.properties delete mode 100644 streampark-spark/streampark-spark-cli/pom.xml create mode 100644 streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/java/org/apache/streampark/spark/configuration/SparkOptions.java rename streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/entity/UserManagementUserType.java => streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/java/org/apache/streampark/spark/configuration/package-info.java (87%) create mode 100644 streampark-spark/streampark-spark-core/src/main/java/org/apache/streampark/spark/configuration/SparkRuntimeOptions.java create mode 100644 streampark-spark/streampark-spark-core/src/main/java/org/apache/streampark/spark/configuration/package-info.java delete mode 100644 streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/ParameterTool.java delete mode 100644 streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SQLContextUtil.scala diff --git a/.asf.yaml b/.asf.yaml index edb44d0c26..b4c1e3e73e 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -16,16 +16,17 @@ # github: - description: Make stream processing easier! Easy-to-use streaming application development framework and operation platform. + description: The Open Control Plane for Real-Time Data homepage: https://streampark.apache.org/ labels: - streaming - - operation-platform - - development-framework - - easy-to-use - - etl-pipeline - - apache - - streampark + - data-computing + - control-plane + - flink + - spark + - lakehouse + - cloud-native + - kubernetes enabled_merge_buttons: squash: true merge: false @@ -37,7 +38,6 @@ github: strict: false contexts: - Backend - Result - - E2E - Result - Docs - Result - Frontend - Result - Unit-Test - Result diff --git a/.github/actions/workflow-telemetry-action b/.github/actions/workflow-telemetry-action deleted file mode 160000 index f974e0c594..0000000000 --- a/.github/actions/workflow-telemetry-action +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f974e0c5942f8f37973c4cc395704165fbe629ba diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index ce0732c004..dd48b47fd2 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -37,9 +37,6 @@ labelPRBasedOnFilePath: component=streampark-console-webapp: - streampark-console/streampark-console-webapp/** - component=streampark-e2e: - - streampark-e2e/**/* - component=streampark-flink: - streampark-flink/**/* diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml deleted file mode 100644 index d2108163b8..0000000000 --- a/.github/workflows/e2e.yml +++ /dev/null @@ -1,252 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -on: - pull_request: - push: - branches: - - dev - - release-* - - dev-* - -name: E2E - -concurrency: - group: e2e-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - - -jobs: - paths-filter: - name: E2E-Path-Filter - runs-on: ubuntu-latest - outputs: - not-ignore: ${{ steps.filter.outputs.not-ignore }} - steps: - - uses: actions/checkout@v4 - - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d - id: filter - with: - filters: | - not-ignore: - - '!**/*.md' - license-header: - name: License header - runs-on: ubuntu-latest - if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} - needs: paths-filter - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Check license header - uses: apache/skywalking-eyes/header@main - code-style: - if: github.repository == 'apache/streampark' - name: Code style - runs-on: ubuntu-latest - needs: license-header - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Check codestyle - run: ./mvnw -B -q -nsu checkstyle:check spotless:check - build: - name: E2E-Build - runs-on: ubuntu-latest - needs: code-style - timeout-minutes: 20 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - java-version: 11 - distribution: 'adopt' - - name: Cache local Maven repository - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e - restore-keys: ${{ runner.os }}-maven- - - name: Build Image - run: | - ./build.sh \ - && docker build -f docker/Dockerfile . -t apache/streampark:ci - - name: Export Docker Images - run: | - docker save apache/streampark:ci -o /tmp/standalone-image.tar \ - && du -sh /tmp/standalone-image.tar - - uses: actions/upload-artifact@v4 - name: Upload Docker Images - with: - name: standalone-image - path: /tmp/standalone-image.tar - retention-days: 1 - e2e: - name: ${{ matrix.case.name }} - needs: build - runs-on: ubuntu-latest - timeout-minutes: 180 - strategy: - matrix: - case: - - name: EnvironmentTest - class: org.apache.streampark.e2e.cases.EnvironmentTest - - name: AlarmTest - class: org.apache.streampark.e2e.cases.AlarmTest - - name: UserManagementTest - class: org.apache.streampark.e2e.cases.UserManagementTest - - name: RoleManagementTest - class: org.apache.streampark.e2e.cases.RoleManagementTest - - name: TeamManagementTest - class: org.apache.streampark.e2e.cases.TeamManagementTest - - name: MemberManagementTest - class: org.apache.streampark.e2e.cases.MemberManagementTest - - name: ExternalLinkTest - class: org.apache.streampark.e2e.cases.ExternalLinkTest - - name: YarnQueueTest - class: org.apache.streampark.e2e.cases.YarnQueueTest - - name: TokenManagementTest - class: org.apache.streampark.e2e.cases.TokenManagementTest - - name: UploadManagementTest - class: org.apache.streampark.e2e.cases.UploadManagementTest - - name: ProjectsManagementTest - class: org.apache.streampark.e2e.cases.ProjectsManagementTest - - name: VariableManagementTest - class: org.apache.streampark.e2e.cases.VariableManagementTest - - - name: FlinkOnRemoteClusterDeployTest - class: org.apache.streampark.e2e.cases.Flink120OnRemoteClusterDeployTest - - # - name: FlinkOnYarnClusterDeployTest - # class: org.apache.streampark.e2e.cases.Flink120OnYarnClusterDeployTest - - # - name: FlinkSQLOnYarnTest - # class: org.apache.streampark.e2e.cases.FlinkSQL120OnYarnTest - - #- name: Flink117OnRemoteClusterDeployTest - # class: org.apache.streampark.e2e.cases.Flink117OnRemoteClusterDeployTest - #- name: Flink116OnRemoteClusterDeployTest - # class: org.apache.streampark.e2e.cases.Flink116OnRemoteClusterDeployTest - - # - name: Flink116OnYarnClusterDeployTest - # class: org.apache.streampark.e2e.cases.Flink116OnYarnClusterDeployTest - # - name: Flink117OnYarnClusterDeployTest - # class: org.apache.streampark.e2e.cases.Flink117OnYarnClusterDeployTest - - # - name: FlinkSQL116OnYarnTest - # class: org.apache.streampark.e2e.cases.FlinkSQL116OnYarnTest - # - name: FlinkSQL117OnYarnTest - # class: org.apache.streampark.e2e.cases.FlinkSQL117OnYarnTest - env: - RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} - steps: - - name: Free Disk Space - # refer: https://github.com/jlumbroso/free-disk-space - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - dotnet: true - android: true - haskell: true - codeql: true - docker-images: true - large-packages: true - - uses: actions/checkout@v4 - with: - submodules: true - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - java-version: 11 - distribution: 'adopt' - - name: Collect Workflow Telemetry - uses: ./.github/actions/workflow-telemetry-action - with: - comment_on_pr: false - - name: Cache local Maven repository - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e - restore-keys: ${{ runner.os }}-maven- - - uses: actions/download-artifact@v4 - name: Download Docker Images - with: - name: standalone-image - path: /tmp - - name: Load Docker Images - run: | - docker load -i /tmp/standalone-image.tar \ - && rm -f /tmp/standalone-image.tar - - name: Pre-pull Testcontainers images - run: | - pull_with_retry() { - local image="$1" - for attempt in $(seq 1 8); do - if docker pull "$image"; then - return 0 - fi - echo "Failed to pull $image (attempt $attempt), retrying in ${attempt}0s..." - sleep $((attempt * 10)) - done - echo "Failed to pull $image after 8 attempts" - return 1 - } - pull_with_retry testcontainers/ryuk:0.7.0 - pull_with_retry selenium/standalone-chrome:4.13.0 || true - - name: Run Test - run: | - ./mvnw -B -f streampark-e2e/pom.xml -am \ - -DfailIfNoTests=false \ - -Dtest=${{ matrix.case.class }} test - - uses: actions/upload-artifact@v4 - if: always() - name: Upload Recording - with: - name: recording-${{ matrix.case.name }} - path: ${{ env.RECORDING_PATH }} - retention-days: 1 - - uses: actions/upload-artifact@v4 - if: always() - name: Upload Projects Build Logs - with: - name: ${{ matrix.case.name }}-projects-build-logs - path: ~/streampark_build_logs - retention-days: 1 - result: - name: E2E - Result - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [ paths-filter, license-header, code-style, build, e2e ] - if: always() - steps: - - name: Status - run: | - if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip E2E!" - exit 0 - fi - if [[ ${{ needs.e2e.result }} != 'success' ]]; then - echo "E2E Failed!" - exit 1 - fi diff --git a/.gitignore b/.gitignore index 0f5129ad83..b6aa543c02 100644 --- a/.gitignore +++ b/.gitignore @@ -47,3 +47,4 @@ hs_err_pid* **/node dist/ dist-license-check/ +.pnpm-store diff --git a/AGENTS.md b/AGENTS.md index 64fe4ca068..4f140b584b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,53 +25,67 @@ Project conventions, architecture, and coding patterns for the StreamPark codeba ### Module Boundaries -StreamPark is a Maven multi-module project with four top-level modules. Each has a clear responsibility boundary. +StreamPark is a Maven multi-module project with five top-level reactor modules. Each has a clear responsibility boundary. -- **`streampark-common`** (`streampark-common/`): Shared foundation layer. Contains configuration management (`ConfigKeys`, `ConfigOption`), utility classes (`Utils`, `HadoopUtils`, `JsonUtils`, `YarnUtils`), file system abstraction (`FsOperator`, `HdfsOperator`, `LfsOperator`), and shared enums (`FlinkDeployMode`, `ApplicationType`, etc.). Must be engine-agnostic — no Flink or Spark core API dependencies. All other modules depend on this module. +- **`streampark-common`** (`streampark-common/`): Engine-API-free foundation layer. It owns the immutable configuration model (`ConfigOption`, `Configuration`, `ConfigurationLoader`, and `GlobalConfiguration`), shared option catalogs, workspace layout, utilities, file system abstractions, and common enums. It may understand external formats such as Flink YAML, but must not depend on Flink or Spark runtime APIs. -- **`streampark-flink`** (`streampark-flink/`): Flink development framework and runtime integration. Contains the core development API (`FlinkStreaming`, `FlinkTable`, `FlinkSQL`), version-specific shims layers (`streampark-flink-shims_flink-1.xx`), job submission clients (`streampark-flink-client`), Kubernetes integration (`streampark-flink-kubernetes`), application packer (`streampark-flink-packer`), and connectors. The shims proxy (`FlinkShimsProxy`) is the central mechanism for multi-version Flink support. +- **`streampark-scala`** (`streampark-scala/`): Small Scala compatibility layer containing shared Scala logging and implicit utilities. Java-only common code belongs in `streampark-common`; reusable Scala code belongs here instead of being embedded in an engine module. -- **`streampark-spark`** (`streampark-spark/`): Spark development framework. Optional module, activated via `-Pspark` Maven profile. Contains `SparkStreaming`, `SparkBatch` core traits, Spark SQL client, and connectors. Follows the same lifecycle pattern as Flink modules. +- **`streampark-flink`** (`streampark-flink/`): Flink runtime integration. Its reactor contains the version-isolated shims, SQL client, submission client, packer, and Kubernetes integration. Connector modules are enabled by the module's `shaded` profile. `FlinkShimsProxy` in the client API is the entry point for executing version-specific code. -- **`streampark-console`** (`streampark-console/`): Web management platform. Contains two submodules: - - **`streampark-console-service`**: Spring Boot 2.7 backend, with `base/` (infrastructure), `core/` (business logic, controllers, services), and `system/` (authentication, user/role/team management) packages. - - **`streampark-console-webapp`**: Vue 3 + Vite frontend, with `api/` (API layer), `views/` (pages), `components/` (reusable UI), `store/` (Pinia state). +- **`streampark-spark`** (`streampark-spark/`): Spark runtime integration. It contains Spark configuration and SQL utilities, submission client API/core, and SQL client. It is a regular root reactor module; the retained `spark` profile is not required to include it in a normal build. -The `streampark-common` module has the strongest stability guarantees — changes here affect all other modules. `streampark-flink/streampark-flink-core` public API (the `FlinkStreaming`, `FlinkTable` traits) should also be treated as stable — breaking changes require careful migration planning. +- **`streampark-console`** (`streampark-console/`): Web management platform. Its Maven reactor contains the Spring Boot service. The sibling `streampark-console-webapp/` Vue 3 application is built into the service only when the service's `webapp` profile is enabled; it is not a Maven child module. + +The strongest compatibility contracts are the common configuration keys and value semantics, Flink client request/response types, shims proxy serialization boundary, database schema, and REST API. Changes to these surfaces require cross-module impact analysis and focused compatibility tests. ### High-Sensitivity Areas -- **`FlinkShimsProxy`**: The multi-version classloader isolation mechanism. Uses `ChildFirstClassLoader` to dynamically load version-specific shims JARs. Cached per Flink version. Changes here affect all Flink jobs across all versions. Never introduce static state that could leak across classloader boundaries. +- **Configuration model** (`org.apache.streampark.common.configuration`): `ConfigOption` declares typed metadata, `ConfigurationParser` parses raw documents, `ConfigurationLoader` composes ordered sources, and immutable `Configuration` snapshots perform typed reads. `GlobalConfiguration` is the process-boundary atomic reference, not a mutable property bag. Option keys and fallback keys are user-facing contracts. + +- **Configuration ownership**: Common option catalogs contain only settings owned by common infrastructure. Console and engine settings stay in their owning modules. `SpringConfigurationInitializer` is the Console composition root and owns the explicit list of options bound from Spring; do not add global `ALL` registries to option catalogs. + +- **Workspace initialization**: `Workspace.LOCAL`, `Workspace.REMOTE`, and derived constants are initialized from one immutable snapshot. Spring bootstrap must publish its completed configuration before workspace paths are first accessed, and must retain the `Workspace.verifyInitializedFrom` guard. -- **`FlinkStreaming` / `FlinkTable` lifecycle**: The `main` -> `init` -> `ready` -> `handle` -> `destroy` lifecycle is the contract all user applications depend on. Changes to the execution order or initialization behavior can break existing applications in production. +- **Flink YAML compatibility** (`FlinkConfigurationUtils`): Before Flink 1.19, load only `flink-conf.yaml` with the legacy line parser. Flink 1.19 and 1.20 support both names and prefer `flink-conf.yaml` when both exist; parser selection follows the selected filename. Flink 2.0 and later load only `config.yaml` with standard nested YAML parsing. Directory-based loading must always receive the target Flink version. This common utility returns engine-neutral maps; conversion to Flink's `Configuration` belongs in a Flink module. -- **`ConfigKeys` / `CommonConfig`**: Central configuration key definitions. Adding, removing, or renaming keys affects application configuration files, the console UI, and deployment scripts. Key names must remain backward-compatible. +- **`FlinkShimsProxy`**: The multi-version classloader isolation mechanism uses `ChildFirstClassLoader` and serializes request/response objects across the boundary. Classloaders are cached by concrete Flink version. Never pass target-runtime Flink objects into the parent classloader or introduce target-specific static state that can leak across classloaders. + +- **Shims modules**: `streampark-flink-shims-base` contains only contracts and implementation shared by every supported shim. Supported concrete modules are Flink 1.18, 1.19, 1.20, 2.0, 2.1, 2.2, and 2.3. APIs removed from Flink 2.x, such as legacy `registerDataStream` and table-function `registerFunction` overloads, belong only in the compatible 1.x implementations. Do not add classes or methods anywhere under `streampark-flink-shims` without an explicit shims architecture decision. + +- **Flink submission flow**: `FlinkClient` and its request/response packages form the stable API. `FlinkClientEntrypoint`, `SubmitRequestResolver`, `SubmissionConfigurationBuilder`, and deployment-specific clients own request normalization, configuration assembly, and submission. Keep Flink-version calls behind the shims boundary. - **`FlinkApplicationController` / `FlinkApplicationManageService` / `FlinkApplicationActionService`**: The core application management flow. Operations (start, stop, cancel, deploy) must be idempotent and handle all Flink states correctly. The `AppChangeEvent` annotation triggers state synchronization. -- **SQL parsing and validation** (`FlinkSql`, `FlinkSqlService`, `SqlConvertUtils`): SQL validation must be version-aware (Flink 1.12-1.20 have different SQL syntax). The `sql-rev.dict` file handles MySQL-to-PostgreSQL dialect conversion. +- **Persistence entities** (`console/core/entity`): These classes map database rows. Keep configuration discovery, parsing, filesystem access, and other business logic in assemblers, services, or utility classes such as `FlinkEnvUtils` and `FlinkApplicationConfigUtils`. Existing small mapping operations may remain, but new domain workflows must not be added to entities. + +- **SQL parsing and validation**: Flink SQL validation is version-aware and executes through the selected 1.18-2.3 shim. `FlinkSql`, `FlinkSqlService`, and the SQL client have different persistence, orchestration, and execution responsibilities. `sql-rev.dict` handles database SQL differences between MySQL and PostgreSQL; it is unrelated to Flink SQL syntax compatibility. -- **Kubernetes integration** (`FlinkK8sWatchController`): Uses Caffeine caches (`TrackIdCache`, `JobStatusCache`, `MetricCache`) for tracking K8s-deployed Flink jobs. Cache invalidation and TTL must be correct to avoid stale state. +- **Kubernetes integration** (`FlinkKubernetesWatchController`): Uses Caffeine caches (`TrackIdCache`, `JobStatusCache`, `MetricCache`) for tracking K8s-deployed Flink jobs. Cache invalidation and TTL must be correct to avoid stale state. -- **Database schema changes**: All schema changes must have corresponding upgrade scripts in `streampark-console/.../script/upgrade/` for both MySQL and PostgreSQL. The `sql-rev.dict` file must be updated if new SQL dialect differences are introduced. +- **HTTP client and proxying**: `OkHttpUtils` in common owns the shared connection pool, timeouts, and bounded retries for idempotent requests. Console proxy code in `WebUtils` owns servlet adaptation, hop-by-hop header filtering, response streaming, and response closure. Do not create ad hoc clients per request or retry non-idempotent methods implicitly. + +- **Database schema changes**: All schema changes must have corresponding upgrade scripts under `streampark-console/streampark-console-service/src/main/assembly/script/upgrade/` for both `mysql/` and `pgsql/`. Update `sql-rev.dict` when a mapper or initialization statement needs a MySQL-to-PostgreSQL rewrite. - **Authentication & Authorization**: `ShiroConfig`, `JWTUtil`, `ShiroRealm` — changes here affect all user access. The `@Permission` annotation and `PermissionAspect` enforce team-level resource isolation. Never weaken RBAC checks. ## Design Patterns -- **Lifecycle trait pattern**: `FlinkStreaming`, `FlinkTable`, `SparkStreaming`, `SparkBatch` all follow the same trait-based lifecycle: `main` -> `init` -> `ready` -> `handle` -> `start` -> `destroy`. Users override `handle()` (required) and optionally `ready()`, `config()`, `destroy()`. Never add mandatory lifecycle methods to existing traits. +- **Immutable configuration pipeline**: Declare typed options in the owning module, parse external input without a central registry, compose sources by precedence, and capture one immutable snapshot at the start of a multi-step operation. Do not repeatedly read mutable global state inside a workflow. + +- **Shims / Proxy pattern**: `FlinkShimsProxy.proxy(flinkVersion, function)` loads the target installation and matching shim behind a child-first classloader. Shared behavior stays in shims-base; version-only API calls stay in the concrete version module. Cross-boundary values must use stable serializable StreamPark request/response types. -- **Shims / Proxy pattern**: `FlinkShimsProxy.proxy(flinkVersion, func)` isolates version-specific Flink API calls behind a `ChildFirstClassLoader`. Each Flink version has its own shims module (`streampark-flink-shims_flink-1.xx`) with the same interface. New shims methods must be added to all version modules. +- **Initializer pattern**: `FlinkStreamInitializer` and `FlinkTableInitializer` assemble application configuration namespaces and construct Flink environments inside the target classloader. Native `flink.property.*`, user `app.*`, and command-line sources remain distinct until their documented composition point. - **Service layer separation**: Console services are split by responsibility — `FlinkApplicationManageService` (CRUD), `FlinkApplicationActionService` (start/stop/cancel), `FlinkApplicationInfoService` (query/info). Follow this pattern when adding new application operations. -- **Implicit enrichment**: Scala `implicit` conversions are used to extend Flink/Spark APIs (e.g., `DataStreamExt` adds methods to `DataStream`). New implicit conversions must be scoped to avoid polluting the global namespace. +- **Scala compatibility layer**: Shared Scala helpers live in `streampark-scala`. New implicits must be narrowly scoped and must not introduce engine dependencies into the common foundation. -- **MyBatis-Plus entity pattern**: Entities extend `BaseEntity` (auto-fill `createTime`/`modifyTime`). Mappers extend MyBatis-Plus `BaseMapper`. Pagination uses `MybatisPager` + `PaginationInterceptor`. Follow existing patterns rather than introducing new ORM approaches. +- **MyBatis-Plus entity pattern**: Mappers extend MyBatis-Plus `BaseMapper`; entities use `@TableName` and related mapping annotations. Only entities that need the shared audit fields extend `BaseEntity`. Pagination uses `MybatisPager` and `PaginationInterceptor`. Keep entities focused on persistence data. -- **Enum-based configuration**: Both Java enums (`FlinkDeployMode`, `ApplicationType`) and Scala enumeratum enums (`ApiType`, `PlannerType`) are used. Prefer Scala `enumeratum` for new Scala-side enums — it provides better type safety and serialization. +- **Typed request and option APIs**: Prefer typed enums, request objects, `ConfigOption`, and immutable maps over unstructured string bags. Preserve serialized field names across the Console-to-client and client-to-shims boundaries. -- **REST response pattern**: All controller methods return `RestResponse.success(data)` or `RestResponse.fail(...)`. Never return raw objects. Use `@Permission` annotation for access control, `@AppChangeEvent` for state-change auditing. +- **REST response pattern**: Normal API endpoints return `RestResponseBody` created through `RestResponseBody.success(...)` or `RestResponseBody.fail(...)`. The map-based `RestResponse` is a deprecated compatibility type and must not be introduced at new controller boundaries. Streaming proxy endpoints may write directly to `HttpServletResponse`. Use `@Permission` for protected resources and `@AppChangeEvent` for audited application changes. - **File system abstraction**: Use `FsOperator` (with `HdfsOperator` / `LfsOperator` implementations) for file operations. Never use raw `java.io.File` or Hadoop `FileSystem` directly in business logic. @@ -84,9 +98,10 @@ The `streampark-common` module has the strongest stability guarantees — change - **Static checks**: Checkstyle (`tools/checkstyle/checkstyle.xml`) + Spotless. No wildcard imports. No `@author` tags. No JUnit 4 imports. - **Lombok**: Use `@Slf4j`, `@Data`, `@Builder` where appropriate. Do not use `@EqualsAndHashCode` on JPA/Hibernate entities. - **Testing**: JUnit 5 (`org.junit.jupiter`) + AssertJ. Use `@Test` (not `@Test` from JUnit 4). Use `assertThat(...).isEqualTo(...)` style. Test classes should be in the same package as the code under test. +- **Method names**: New production and test method names must be concise, readable, and no longer than 40 characters. - **Package structure**: Controllers in `controller/`, service interfaces in `service/`, implementations in `service/impl/`, entities in `entity/`, mappers in `mapper/`, enums in `enums/`. -### Scala (Framework / Common) +### Scala (Scala Support / Spark) - **Formatting**: Scalafmt 3.7.5 (`tools/checkstyle/.scalafmt.conf`). Max column 160. Run `./mvnw spotless:apply` to format. - **Import ordering**: `org.apache.streampark.*` first, then other third-party, then `javax.*`, `java.*`, `scala.*`. @@ -123,24 +138,29 @@ The `streampark-common` module has the strongest stability guarantees — change ./mvnw -Pfast clean install -DskipTests ``` -- **Build with Spark module:** +- **Build backend reactor with shaded artifacts:** ```shell - ./mvnw -Pspark,shaded clean install -DskipTests + ./mvnw -Pshaded clean install -DskipTests ``` - **Build backend only:** ```shell - ./mvnw clean install -DskipTests -pl '!streampark-console/streampark-console-webapp' + ./mvnw clean install -DskipTests ``` - **Run single test class (Java):** ```shell - ./mvnw test -pl streampark-console/streampark-console-service -Dtest=FlinkApplicationControllerTest + ./mvnw test -pl streampark-console/streampark-console-service -Dtest=FlinkSavepointServiceTest + ``` + +- **Run common configuration tests:** + ```shell + ./mvnw test -pl streampark-common -Dtest=ConfigurationTest,FlinkConfigurationUtilsTest ``` -- **Run single test class (Scala):** +- **Build all supported Flink shims:** ```shell - ./mvnw test -pl streampark-common -Dtest=org.apache.streampark.common.util.CommandUtilsTest + ./mvnw -f streampark-flink/streampark-flink-shims/pom.xml clean install ``` - **Format code (Java + Scala):** @@ -185,8 +205,8 @@ The `streampark-common` module has the strongest stability guarantees — change - **One concern per PR**: Unrelated whitespace, import, or formatting changes go in separate PRs. Do not mix refactoring with feature work. - **Commit messages**: Describe the *what* and *why*, not implementation details. Reference related GitHub issues with `#xxx`. - **Apache License header**: Required on all new files (enforced by Spotless and Apache RAT). The `spotless:check` and `apache-rat:check` goals run in CI. -- **Schema changes**: Must include upgrade scripts for both MySQL and PostgreSQL in `streampark-console/.../script/upgrade/`. -- **New shims methods**: When adding a new method to the shims interface, add implementations to all version-specific shims modules (`streampark-flink-shims_flink-1.12` through `streampark-flink-shims_flink-1.20`). +- **Schema changes**: Must include matching scripts for MySQL and PostgreSQL under `streampark-console/streampark-console-service/src/main/assembly/script/upgrade/`. +- **Shims changes**: Keep shared methods in shims-base only when every supported version can implement them. Put version-specific Flink APIs only in the applicable 1.18-2.3 concrete modules, and verify the full shims reactor. ## Boundaries @@ -194,9 +214,9 @@ The `streampark-common` module has the strongest stability guarantees — change - **Never** modify `.asf.yaml`, `LICENSE`, `NOTICE`, or `.gitignore` without explicit discussion. - **Never** upgrade Flink, Spark, Scala, Spring Boot, or other major dependency versions without discussion — these changes have broad impact across the entire project. -- **Never** remove or rename keys in `ConfigKeys` or `CommonConfig` — these are user-facing configuration contracts. -- **Never** change the `FlinkStreaming` / `FlinkTable` lifecycle method signatures — this breaks all user applications. -- **Never** add new required lifecycle methods to the `FlinkStreaming` / `FlinkTable` traits. +- **Never** remove or rename canonical or fallback configuration keys without an explicit compatibility decision; they are user-facing contracts. +- **Never** add classes or methods under `streampark-flink-shims` without an explicit architecture decision. General utilities, Console logic, and configuration-file parsing belong outside shims. +- **Never** add business workflows or infrastructure access to `streampark-console-service/.../core/entity`. - **Never** commit secrets, credentials, API keys, or cloud-specific tokens. - **Never** introduce a new Flink version shims module without adding the corresponding CI build configuration. - **Never** add a new database migration without providing both MySQL and PostgreSQL upgrade scripts. @@ -207,5 +227,5 @@ The `streampark-common` module has the strongest stability guarantees — change - **Ask first** before adding new third-party dependencies — license compatibility with Apache 2.0 matters. - **Ask first** before promoting package-private classes/methods to public. - **Ask first** before adding new Maven modules or restructuring the module hierarchy. -- **Ask first** before introducing new Scala implicits in the `common` module — they affect all downstream code. -- **Ask first** before changing the authentication model (Shiro/JWT/Pac4j configuration). \ No newline at end of file +- **Ask first** before introducing new Scala implicits in `streampark-scala` — they affect all downstream Scala code. +- **Ask first** before changing the authentication model (Shiro/JWT/Pac4j configuration). diff --git a/NOTICE b/NOTICE index 944f638864..621dea7c08 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache StreamPark -Copyright 2022-2025 The Apache Software Foundation +Copyright 2022-2026 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/build.sh b/build.sh index 968936c1b7..0b98c78eb8 100755 --- a/build.sh +++ b/build.sh @@ -163,7 +163,7 @@ print_logo() { printf ' %s ___/ / /_/ / / __/ /_/ / / / / / / /_/ / /_/ / / / ,< %s\n' $PRIMARY $RESET printf ' %s /____/\__/_/ \___/\__,_/_/ /_/ /_/ ____/\__,_/_/ /_/|_| %s\n' $PRIMARY $RESET printf ' %s /_/ %s\n\n' $PRIMARY $RESET - printf ' %s Version: 3.0.0-SNAPSHOT %s\n' $BLUE $RESET + printf ' %s Version: 3.0.0 %s\n' $BLUE $RESET printf ' %s WebSite: https://streampark.apache.org%s\n' $BLUE $RESET printf ' %s GitHub : http://github.com/apache/streampark%s\n\n' $BLUE $RESET printf ' %s ──────── Apache StreamPark, Make stream processing easier ô~ô!%s\n\n' $PRIMARY $RESET diff --git a/dist-material/release-docs/NOTICE b/dist-material/release-docs/NOTICE index 944f638864..621dea7c08 100644 --- a/dist-material/release-docs/NOTICE +++ b/dist-material/release-docs/NOTICE @@ -1,5 +1,5 @@ Apache StreamPark -Copyright 2022-2025 The Apache Software Foundation +Copyright 2022-2026 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/pom.xml b/pom.xml index 9f90652f93..c78dceb096 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ streampark-common - streampark-common-scala-bridge + streampark-scala streampark-flink streampark-spark streampark-console @@ -83,7 +83,7 @@ - 3.0.0-SNAPSHOT + 3.0.0 11 UTF-8 UTF-8 @@ -98,20 +98,14 @@ 1.20.1 1.8.1 1.0.0 - 1.17 + 1.18 3.1.2 4.0.0 - 2.3.4 - 3.3.4 + 3.3.5 2.1.10 - 3.3.0 - 3.6.3 - 6.2.3 - 2.17 2.5.0 1.7.32 1.2.11 - 1.15.0 2.13.5 30.0-jre 2.8.6 @@ -143,6 +137,8 @@ 1.6.0 org.apache.streampark.shaded 5.1 + 4.11.0 + 3.5.0 1.18.24 5.9.1 3.4.6 @@ -228,7 +224,7 @@ org.apache.streampark - streampark-common-scala-bridge_${scala.binary.version} + streampark-scala_${scala.binary.version} ${project.version} @@ -251,18 +247,6 @@ test - - org.apache.zookeeper - zookeeper - ${zoopkeeper.version} - - - - redis.clients - jedis - ${redis.version} - - com.zaxxer HikariCP @@ -354,6 +338,18 @@ ${httpclient5.version} + + com.squareup.okhttp3 + okhttp + ${okhttp.version} + + + + com.squareup.okio + okio + ${okio.version} + + org.apache.hadoop hadoop-client-api @@ -496,25 +492,6 @@ force-shading - - - org.scala-lang - scala-library - provided - - - - org.scala-lang - scala-compiler - provided - - - - org.scala-lang - scala-reflect - provided - - @@ -631,8 +608,6 @@ src/main/java/**/*.java src/test/java/**/*.java - streampark-e2e/streampark-e2e-case/src/test/java/**/*.java - streampark-e2e/streampark-e2e-core/src/main/java/**/*.java ${maven.multiModuleProjectDirectory}/tools/checkstyle/spotless_streampark_formatter.xml diff --git a/streampark b/streampark new file mode 100755 index 0000000000..be6573120d --- /dev/null +++ b/streampark @@ -0,0 +1,510 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +MAIN_CLASS="org.apache.streampark.console.StreamParkConsoleBootstrap" +BASH_UTIL_CLASS="org.apache.streampark.console.base.util.BashJavaUtils" +DEFAULT_DIST="dist/apache-streampark-3.0.0-bin" +SERVICE_MODULE="streampark-console/streampark-console-service" +BUILD_MODULES="streampark-common,streampark-flink/streampark-flink-client/streampark-flink-client-api,streampark-flink/streampark-flink-client/streampark-flink-client-core,$SERVICE_MODULE" + +usage() { + cat <<'USAGE' +Usage: + streampark [command] + streampark [flag] [command] + +Default: + streampark + Same as "streampark start". Build current source, then start streampark from + target/classes in the background. If an streampark dev process is already + running, it is restarted after the build succeeds. + +Commands: + build + Compile current source with: + ./mvnw -pl streampark-common,streampark-flink-client-api, + streampark-flink-client-core,streampark-console-service -DskipTests compile + This matches the IDEA development loop and does not package dist. + + start + Build current source, then start the platform in the background. + Runtime app.home still points to dist for conf/log/license/temp/JDK, + but platform classes are loaded from current target/classes first. + + run + Build current source, then start the platform in the foreground. + Use this when you want logs directly in the terminal. + + stop + Stop the background dev process started by "streampark" or "streampark start". + + restart + Stop the background dev process, build current source, then start it + again in the background. + + status + Show whether the background dev process is running. + + logs | log + Tail the background dev log: + $STREAMPARK_APP_HOME/logs/streampark-dev.log + + classpath + Build current source and print the effective development classpath. + Source module target/classes entries are placed before dependency jars. + + info + Print resolved paths: repo.home, app.home, java.home, pid.file, + log.file, and main.class. + + help | --help | -h + Print this help. + +Flags: + --no-build CMD + Run start, run, or classpath without compiling first. Useful when the + code was already compiled by IDEA or a previous "streampark build". + + --no-compile CMD + Alias for --no-build. + +Environment: + STREAMPARK_APP_HOME + Runtime app.home directory. + Default: repo/dist/apache-streampark-3.0.0-bin + + STREAMPARK_JAVA_HOME + Java home for the platform process and build. + Default: bundled JDK, JAVA_HOME, macOS JDK 11, or java from PATH. + + STREAMPARK_JAVA_OPTS + Extra JVM options for the platform process. + Example: STREAMPARK_JAVA_OPTS="-Xms1g -Xmx4g" + + STREAMPARK_MVN_OPTS + Extra Maven options appended to build and classpath commands. + Example: STREAMPARK_MVN_OPTS="-o" + + STREAMPARK_STARTUP_TIMEOUT + Seconds to wait for the Spring Boot ready log. Default: 60. + +Examples: + streampark + Build latest code and deploy it in the background. + + streampark build + Only compile latest code. + + streampark --no-build start + Restart from existing target/classes without compiling. + + STREAMPARK_JAVA_OPTS="-Xms1g -Xmx4g" streampark start + Start with extra JVM memory options. +USAGE +} + +resolve_script() { + local source="${BASH_SOURCE[0]}" + while [[ -L "$source" ]]; do + local dir + dir="$(cd -P "$(dirname "$source")" >/dev/null 2>&1 && pwd)" + source="$(readlink "$source")" + [[ "$source" != /* ]] && source="$dir/$source" + done + cd -P "$(dirname "$source")" >/dev/null 2>&1 && pwd +} + +SCRIPT_DIR="$(resolve_script)" +REPO_HOME="$SCRIPT_DIR" +APP_HOME="${STREAMPARK_APP_HOME:-$REPO_HOME/$DEFAULT_DIST}" +MVNW="$REPO_HOME/mvnw" +CACHE_DIR="$REPO_HOME/target/dev" +DEPENDENCY_CP_FILE="$CACHE_DIR/streampark-console-runtime.cp" +PID_FILE="$APP_HOME/.streampark-dev.pid" +LOG_FILE="$APP_HOME/logs/streampark-dev.log" +LOCK_DIR="$APP_HOME/.streampark-dev.lock" +export APP_LOG="$APP_HOME/logs" + +if [[ ! -x "$MVNW" ]]; then + echo "Cannot find executable Maven wrapper: $MVNW" >&2 + exit 1 +fi + +if [[ ! -d "$APP_HOME" ]]; then + echo "Cannot find streampark app.home: $APP_HOME" >&2 + exit 1 +fi + +if [[ -n "${STREAMPARK_JAVA_HOME:-}" ]]; then + JAVA_HOME="$STREAMPARK_JAVA_HOME" +elif [[ -x "$APP_HOME/opt/jdk/bin/java" ]]; then + JAVA_HOME="$APP_HOME/opt/jdk" +elif [[ -n "${JAVA_HOME:-}" && -x "$JAVA_HOME/bin/java" ]]; then + JAVA_HOME="$JAVA_HOME" +elif [[ -x /usr/libexec/java_home ]]; then + JAVA_HOME="$(/usr/libexec/java_home -v 11 2>/dev/null || /usr/libexec/java_home 2>/dev/null || true)" +elif command -v java >/dev/null 2>&1; then + JAVA_HOME="$(java -XshowSettings:properties -version 2>&1 \ + | awk -F'= ' '/^[[:space:]]*java.home = / { print $2; exit }')" +else + JAVA_HOME="" +fi + +if [[ -z "$JAVA_HOME" ]]; then + echo "Cannot find a JDK. Set STREAMPARK_JAVA_HOME or JAVA_HOME." >&2 + exit 1 +fi +export JAVA_HOME + +JAVA_CMD="$JAVA_HOME/bin/java" +if [[ ! -x "$JAVA_CMD" ]]; then + echo "Cannot execute Java: $JAVA_CMD" >&2 + exit 1 +fi + +compile_sources() { + echo "[streampark] Compiling current source from $REPO_HOME" + ( + cd "$REPO_HOME" + # shellcheck disable=SC2086 + "$MVNW" -pl "$BUILD_MODULES" \ + -DskipTests \ + -Dcheckstyle.skip=true \ + -Dspotless.skip=true \ + -Dscalastyle.skip=true \ + -Drat.skip=true \ + ${STREAMPARK_MVN_OPTS:-} \ + compile + ) +} + +refresh_dependency_classpath() { + mkdir -p "$CACHE_DIR" + local needs_refresh="false" + if [[ ! -s "$DEPENDENCY_CP_FILE" ]]; then + needs_refresh="true" + elif [[ "$SCRIPT_DIR/streampark" -nt "$DEPENDENCY_CP_FILE" ]]; then + needs_refresh="true" + elif find "$REPO_HOME" -path "$REPO_HOME/dist" -prune -o -name pom.xml -newer "$DEPENDENCY_CP_FILE" -print -quit | grep -q .; then + needs_refresh="true" + fi + + if [[ "$needs_refresh" == "true" ]]; then + echo "[streampark] Refreshing Maven dependency classpath" + ( + cd "$REPO_HOME" + # shellcheck disable=SC2086 + # Resolve reactor projects so stale local SNAPSHOT POMs cannot alter the runtime classpath. + "$MVNW" -q -pl "$SERVICE_MODULE" -am \ + -DskipTests \ + -Dcheckstyle.skip=true \ + -Dspotless.skip=true \ + -Dscalastyle.skip=true \ + ${STREAMPARK_MVN_OPTS:-} \ + dependency:build-classpath \ + -DincludeScope=runtime \ + -Dmdep.outputFile="$DEPENDENCY_CP_FILE" + ) + fi +} + +module_classpath() { + local paths=( + "$REPO_HOME/streampark-console/streampark-console-service/target/classes" + "$REPO_HOME/streampark-common/target/classes" + "$REPO_HOME/streampark-flink/streampark-flink-client/streampark-flink-client-api/target/classes" + "$REPO_HOME/streampark-flink/streampark-flink-client/streampark-flink-client-core/target/classes" + ) + + local existing=() + local path + for path in "${paths[@]}"; do + [[ -d "$path" ]] && existing+=("$path") + done + + if [[ "${#existing[@]}" -gt 0 ]]; then + local IFS=: + printf '%s' "${existing[*]}" + fi +} + +filtered_dependency_classpath() { + if [[ ! -s "$DEPENDENCY_CP_FILE" ]]; then + return 0 + fi + + # SQL clients are standalone launch artifacts under the distribution's client/ directory. + tr ':' '\n' < "$DEPENDENCY_CP_FILE" \ + | grep -v '/org/apache/streampark/streampark-console-service/' \ + | grep -v '/org/apache/streampark/streampark-common/' \ + | grep -v '/org/apache/streampark/streampark-flink-client-api/' \ + | grep -v '/org/apache/streampark/streampark-flink-client-core/' \ + | grep -v '/org/apache/streampark/streampark-flink-sqlclient/' \ + | grep -v '/org/apache/streampark/streampark-spark-sqlclient_' \ + | paste -sd ':' - +} + +build_classpath() { + local module_cp dependency_cp + module_cp="$(module_classpath)" + dependency_cp="$(filtered_dependency_classpath)" + + printf '.:%s/lib:%s/jre/lib' "$JAVA_HOME" "$JAVA_HOME" + [[ -n "$module_cp" ]] && printf ':%s' "$module_cp" + [[ -n "$dependency_cp" ]] && printf ':%s' "$dependency_cp" +} + +build_java_command() { + local cp="$1" + STREAMPARK_JAVA_COMMAND=("$JAVA_CMD") + + if [[ -n "${STREAMPARK_JAVA_OPTS:-}" ]]; then + # shellcheck disable=SC2206 + local extra_opts=( ${STREAMPARK_JAVA_OPTS} ) + STREAMPARK_JAVA_COMMAND+=("${extra_opts[@]}") + fi + + local default_opts=( + "-Dstreampark.enterprise=true" + "-Dapp.home=$APP_HOME" + "-DAPP_LOG=$APP_HOME/logs" + "-Dlogging.config=$APP_HOME/conf/logback-spring.xml" + "-Djava.io.tmpdir=$APP_HOME/temp" + ) + + STREAMPARK_JAVA_COMMAND+=("${default_opts[@]}" "-classpath" "$cp" "$MAIN_CLASS") +} + +resolve_server_port() { + "$JAVA_CMD" -cp "$APP_HOME/lib/*" "$BASH_UTIL_CLASS" \ + --get_yaml "server.port" "$APP_HOME/conf/config.yaml" +} + +wait_for_server_port() { + local port="$1" + local used + local i + for i in {1..10}; do + used="$("$JAVA_CMD" -cp "$APP_HOME/lib/*" "$BASH_UTIL_CLASS" --check_port "$port")" + [[ "$used" != "used" ]] && return 0 + sleep 1 + done + echo "[streampark] Cannot start: server port $port is still in use" >&2 + return 1 +} + +is_running() { + [[ -f "$PID_FILE" ]] || return 1 + local pid + pid="$(cat "$PID_FILE" 2>/dev/null || true)" + [[ "$pid" =~ ^[0-9]+$ ]] || return 1 + kill -0 "$pid" >/dev/null 2>&1 +} + +acquire_process_lock() { + if ! mkdir "$LOCK_DIR" 2>/dev/null; then + echo "[streampark] Another start, run, restart, or stop command is in progress" >&2 + exit 1 + fi + trap 'rmdir "$LOCK_DIR" >/dev/null 2>&1 || true' EXIT + trap 'exit 130' INT + trap 'exit 143' TERM +} + +stop_process() { + if ! is_running; then + echo "[streampark] streampark dev process is not running" + rm -f "$PID_FILE" >/dev/null 2>&1 || true + return 0 + fi + + local pid + pid="$(cat "$PID_FILE")" + echo "[streampark] Stopping streampark dev process: $pid" + kill -TERM "$pid" >/dev/null 2>&1 || true + + local i + for i in {1..20}; do + if ! kill -0 "$pid" >/dev/null 2>&1; then + rm -f "$PID_FILE" >/dev/null 2>&1 || true + echo "[streampark] Stopped" + return 0 + fi + sleep 1 + done + + echo "[streampark] Process did not stop after 20s, sending KILL" + kill -KILL "$pid" >/dev/null 2>&1 || true + rm -f "$PID_FILE" >/dev/null 2>&1 || true +} + +prepare() { + local build="$1" + [[ "$build" == "true" ]] && compile_sources + refresh_dependency_classpath + mkdir -p "$APP_HOME/logs/audit" "$APP_HOME/temp" +} + +run_foreground() { + local build="$1" + prepare "$build" + local cp + cp="$(build_classpath)" + echo "[streampark] Starting from current target/classes" + local -a STREAMPARK_JAVA_COMMAND + build_java_command "$cp" + "${STREAMPARK_JAVA_COMMAND[@]}" +} + +run_background() { + local build="$1" + prepare "$build" + local cp + cp="$(build_classpath)" + if is_running; then + echo "[streampark] Existing dev process detected; restarting to deploy current source" + stop_process + fi + echo "[streampark] Starting from current target/classes in background" + echo "[streampark] Log: $LOG_FILE" + local server_port + server_port="$(resolve_server_port)" + if [[ ! "$server_port" =~ ^[0-9]+$ ]]; then + echo "[streampark] Cannot resolve server.port from $APP_HOME/conf/config.yaml" >&2 + return 1 + fi + wait_for_server_port "$server_port" + local info_log="$APP_HOME/logs/info.$(date +%F).log" + local info_start_line=0 + [[ -f "$info_log" ]] && info_start_line="$(wc -l < "$info_log")" + local -a STREAMPARK_JAVA_COMMAND + build_java_command "$cp" + local timeout="${STREAMPARK_STARTUP_TIMEOUT:-60}" + if [[ ! "$timeout" =~ ^[0-9]+$ ]]; then + echo "STREAMPARK_STARTUP_TIMEOUT must be a non-negative integer" >&2 + return 1 + fi + + nohup "${STREAMPARK_JAVA_COMMAND[@]}" >> "$LOG_FILE" 2>&1 & + local pid=$! + echo "$pid" > "$PID_FILE" + + local i + for ((i = 0; i < timeout; i++)); do + if ! kill -0 "$pid" >/dev/null 2>&1; then + rm -f "$PID_FILE" + echo "[streampark] Process exited during startup. Recent log output:" >&2 + tail -n 40 "$LOG_FILE" >&2 + return 1 + fi + if [[ -f "$info_log" ]] && awk -v start="$info_start_line" \ + 'NR > start && /Started StreamParkConsoleBootstrap/ { found = 1 } END { exit !found }' \ + "$info_log"; then + echo "[streampark] Started. PID: $pid" + return 0 + fi + sleep 1 + done + + echo "[streampark] Process is running but readiness was not confirmed within ${timeout}s. PID: $pid" +} + +print_classpath() { + local build="$1" + prepare "$build" + build_classpath + printf '\n' +} + +tail_logs() { + mkdir -p "$(dirname "$LOG_FILE")" + touch "$LOG_FILE" + tail -f "$LOG_FILE" +} + +print_info() { + echo "repo.home=$REPO_HOME" + echo "app.home=$APP_HOME" + echo "java.home=$JAVA_HOME" + echo "pid.file=$PID_FILE" + echo "log.file=$LOG_FILE" + echo "main.class=$MAIN_CLASS" +} + +build_only() { + compile_sources + refresh_dependency_classpath + echo "[streampark] Build complete" +} + +cmd="${1:-start}" +build="true" +if [[ "$cmd" == "--no-build" || "$cmd" == "--no-compile" ]]; then + build="false" + cmd="${2:-start}" +fi + +case "$cmd" in + start|run|restart|stop) + acquire_process_lock + ;; +esac + +case "$cmd" in + build) + build_only + ;; + run) + run_foreground "$build" + ;; + start) + run_background "$build" + ;; + stop) + stop_process + ;; + restart) + stop_process + run_background "$build" + ;; + status) + if is_running; then + echo "streampark dev process is running: $(cat "$PID_FILE")" + else + echo "streampark dev process is not running" + fi + ;; + info) + print_info + ;; + logs|log) + tail_logs + ;; + classpath) + print_classpath "$build" + ;; + -h|--help|help) + usage + ;; + *) + echo "Unknown command: $cmd" >&2 + usage >&2 + exit 1 + ;; +esac diff --git a/streampark-common/pom.xml b/streampark-common/pom.xml index cba450c225..6f23ed9f18 100644 --- a/streampark-common/pom.xml +++ b/streampark-common/pom.xml @@ -29,6 +29,12 @@ + + org.scala-lang + scala-library + provided + + org.junit.jupiter junit-jupiter-engine @@ -49,12 +55,6 @@ test - - redis.clients - jedis - true - - com.zaxxer HikariCP @@ -107,6 +107,13 @@ httpclient5 + + + com.squareup.okhttp3 + okhttp + provided + + io.netty netty-resolver @@ -126,6 +133,7 @@ streampark-shaded-jackson ${streampark.shaded.version} + diff --git a/streampark-common/src/main/java/org/apache/streampark/common/conf/CommonConfig.java b/streampark-common/src/main/java/org/apache/streampark/common/conf/CommonConfig.java deleted file mode 100644 index 65eac9dc64..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/conf/CommonConfig.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.conf; - -/** Common internal configuration options. */ -public final class CommonConfig { - - private CommonConfig() { - } - - private static final InternalOption STREAMPARK_WORKSPACE_LOCAL_OPTION = - new InternalOption("streampark.workspace.local", "/streampark", String.class); - - private static final InternalOption STREAMPARK_WORKSPACE_REMOTE_OPTION = - new InternalOption("streampark.workspace.remote", "/streampark", String.class); - - private static final InternalOption STREAMPARK_HADOOP_USER_NAME_OPTION = - new InternalOption("streampark.hadoop-user-name", "hdfs", String.class); - - private static final InternalOption STREAMPARK_PROXY_YARN_URL_OPTION = - new InternalOption( - "streampark.proxy.yarn-url", - "", - String.class, - "proxy yarn url. ex: knox proxy or other"); - - private static final InternalOption STREAMPARK_YARN_AUTH_OPTION = - new InternalOption( - "streampark.yarn.http-auth", - "", - String.class, - "yarn http auth type. ex: simple, kerberos"); - - private static final InternalOption DOCKER_HOST_OPTION = - new InternalOption( - "streampark.docker.http-client.docker-host", - "", - String.class, - "docker host for DockerHttpClient"); - - private static final InternalOption DOCKER_MAX_CONNECTIONS_OPTION = - new InternalOption( - "streampark.docker.http-client.max-connections", - 100, - Integer.class, - "instantiating max connections for DockerHttpClient"); - - private static final InternalOption DOCKER_CONNECTION_TIMEOUT_SEC_OPTION = - new InternalOption( - "streampark.docker.http-client.connection-timeout-sec", - 100L, - Long.class, - "instantiating connection timeout for DockerHttpClient"); - - private static final InternalOption DOCKER_RESPONSE_TIMEOUT_SEC_OPTION = - new InternalOption( - "streampark.docker.http-client.response-timeout-sec", - 120L, - Long.class, - "instantiating connection timeout for DockerHttpClient"); - - private static final InternalOption MAVEN_SETTINGS_PATH_OPTION = - new InternalOption( - "streampark.maven.settings", - null, - String.class, - "maven settings.xml full path"); - - private static final InternalOption MAVEN_REMOTE_URL_OPTION = - new InternalOption( - "streampark.maven.central.repository", - "https://repo1.maven.org/maven2/", - String.class, - "maven repository used for built-in compilation"); - - private static final InternalOption MAVEN_AUTH_USER_OPTION = - new InternalOption( - "streampark.maven.auth.user", - null, - String.class, - "maven repository used for built-in compilation"); - - private static final InternalOption MAVEN_AUTH_PASSWORD_OPTION = - new InternalOption( - "streampark.maven.auth.password", - null, - String.class, - "maven repository used for built-in compilation"); - - private static final InternalOption KERBEROS_TTL_OPTION = - new InternalOption("security.kerberos.ttl", "2h", String.class, "kerberos default ttl"); - - private static final InternalOption READ_LOG_MAX_SIZE_OPTION = - new InternalOption( - "streampark.read-log.max-size", - "1mb", - String.class, - "The maximum size of the default read log"); - - private static final InternalOption SPRING_PROFILES_ACTIVE_OPTION = - new InternalOption( - "spring.profiles.active", "h2", String.class, "Use the database type"); - - public static InternalOption STREAMPARK_WORKSPACE_LOCAL() { - return STREAMPARK_WORKSPACE_LOCAL_OPTION; - } - - public static InternalOption STREAMPARK_WORKSPACE_REMOTE() { - return STREAMPARK_WORKSPACE_REMOTE_OPTION; - } - - public static InternalOption STREAMPARK_HADOOP_USER_NAME() { - return STREAMPARK_HADOOP_USER_NAME_OPTION; - } - - public static InternalOption STREAMPARK_PROXY_YARN_URL() { - return STREAMPARK_PROXY_YARN_URL_OPTION; - } - - public static InternalOption STREAMPARK_YARN_AUTH() { - return STREAMPARK_YARN_AUTH_OPTION; - } - - public static InternalOption DOCKER_HOST() { - return DOCKER_HOST_OPTION; - } - - public static InternalOption DOCKER_MAX_CONNECTIONS() { - return DOCKER_MAX_CONNECTIONS_OPTION; - } - - public static InternalOption DOCKER_CONNECTION_TIMEOUT_SEC() { - return DOCKER_CONNECTION_TIMEOUT_SEC_OPTION; - } - - public static InternalOption DOCKER_RESPONSE_TIMEOUT_SEC() { - return DOCKER_RESPONSE_TIMEOUT_SEC_OPTION; - } - - public static InternalOption MAVEN_SETTINGS_PATH() { - return MAVEN_SETTINGS_PATH_OPTION; - } - - public static InternalOption MAVEN_REMOTE_URL() { - return MAVEN_REMOTE_URL_OPTION; - } - - public static InternalOption MAVEN_AUTH_USER() { - return MAVEN_AUTH_USER_OPTION; - } - - public static InternalOption MAVEN_AUTH_PASSWORD() { - return MAVEN_AUTH_PASSWORD_OPTION; - } - - public static InternalOption KERBEROS_TTL() { - return KERBEROS_TTL_OPTION; - } - - public static InternalOption READ_LOG_MAX_SIZE() { - return READ_LOG_MAX_SIZE_OPTION; - } - - public static InternalOption SPRING_PROFILES_ACTIVE() { - return SPRING_PROFILES_ACTIVE_OPTION; - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/conf/ConfigKeys.java b/streampark-common/src/main/java/org/apache/streampark/common/conf/ConfigKeys.java deleted file mode 100644 index 919301083e..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/conf/ConfigKeys.java +++ /dev/null @@ -1,396 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.conf; - -/** - * Configuration key constants. - * - *

Parameterless {@code KEY_XXX()} methods mirror the original Scala API for downstream modules. - */ -@SuppressWarnings({"java:S100", "java:S3400"}) -public final class ConfigKeys { - - private ConfigKeys() { - } - - public static String PARAM_PREFIX() { - return "--"; - } - - /** about parameter... */ - public static String KEY_APP_HOME() { - return "app.home"; - } - - public static String KEY_HOST() { - return "host"; - } - - public static String KEY_PORT() { - return "port"; - } - - public static String KEY_DB() { - return "db"; - } - - public static String KEY_USER() { - return "user"; - } - - public static String KEY_PASSWORD() { - return "password"; - } - - public static String KEY_TIMEOUT() { - return "timeout"; - } - - public static String KEY_SEMANTIC() { - return "semantic"; - } - - /** kerberos */ - public static String KEY_KERBEROS() { - return "kerberos"; - } - - public static String KEY_KERBEROS_SERVICE_ACCOUNT() { - return "kubernetes.service-account"; - } - - public static String KEY_HADOOP_USER_NAME() { - return "HADOOP_USER_NAME"; - } - - /** hadoop.security.authentication */ - public static String KEY_HADOOP_SECURITY_AUTHENTICATION() { - return "hadoop.security.authentication"; - } - - public static String KEY_SECURITY_KERBEROS_ENABLE() { - return "security.kerberos.login.enable"; - } - - public static String KEY_SECURITY_KERBEROS_DEBUG() { - return "security.kerberos.login.debug"; - } - - public static String KEY_SECURITY_KERBEROS_KEYTAB() { - return "security.kerberos.login.keytab"; - } - - public static String KEY_SECURITY_KERBEROS_PRINCIPAL() { - return "security.kerberos.login.principal"; - } - - public static String KEY_SECURITY_KERBEROS_KRB5_CONF() { - return "security.kerberos.login.krb5"; - } - - /** about spark */ - public static String KEY_SPARK_MAIN_CLASS() { - return "spark.main.class"; - } - - public static String KEY_SPARK_APP_NAME() { - return "spark.app.name"; - } - - public static String KEY_SPARK_BATCH_DURATION() { - return "spark.batch.duration"; - } - - public static String KEY_SPARK_DRIVER_CORES() { - return "spark.driver.cores"; - } - - public static String KEY_SPARK_DRIVER_MEMORY() { - return "spark.driver.memory"; - } - - public static String KEY_SPARK_EXECUTOR_INSTANCES() { - return "spark.executor.instances"; - } - - public static String KEY_SPARK_EXECUTOR_CORES() { - return "spark.executor.cores"; - } - - public static String KEY_SPARK_EXECUTOR_MEMORY() { - return "spark.executor.memory"; - } - - public static String KEY_SPARK_DYNAMIC_ALLOCATION_ENABLED() { - return "spark.dynamicAllocation.enabled"; - } - - public static String KEY_SPARK_DYNAMIC_ALLOCATION_MAX_EXECUTORS() { - return "spark.dynamicAllocation.maxExecutors"; - } - - public static String KEY_SPARK_YARN_QUEUE() { - return "spark.yarn.queue"; - } - - public static String KEY_SPARK_YARN_QUEUE_NAME() { - return "yarnQueueName"; - } - - public static String KEY_SPARK_YARN_QUEUE_LABEL() { - return "yarnQueueLabel"; - } - - public static String KEY_SPARK_YARN_AM_NODE_LABEL() { - return "spark.yarn.am.nodeLabelExpression"; - } - - public static String KEY_SPARK_YARN_EXECUTOR_NODE_LABEL() { - return "spark.yarn.executor.nodeLabelExpression"; - } - - public static String KEY_SPARK_SQL() { - return KEY_SPARK_SQL(null); - } - - public static String KEY_SPARK_SQL(String prefix) { - return (prefix != null ? prefix : "") + "sql"; - } - - /** about config flink */ - public static String KEY_APP_CONF() { - return KEY_APP_CONF(null); - } - - public static String KEY_APP_CONF(String prefix) { - return (prefix != null ? prefix : "") + "conf"; - } - - public static String KEY_FLINK_CONF() { - return KEY_FLINK_CONF(null); - } - - public static String KEY_FLINK_CONF(String prefix) { - return (prefix != null ? prefix : "") + "flink.conf"; - } - - public static String KEY_APP_NAME() { - return KEY_APP_NAME(null); - } - - public static String KEY_APP_NAME(String prefix) { - return (prefix != null ? prefix : "") + "app.name"; - } - - public static String KEY_FLINK_SQL() { - return KEY_FLINK_SQL(null); - } - - public static String KEY_FLINK_SQL(String prefix) { - return (prefix != null ? prefix : "") + "sql"; - } - - public static String KEY_FLINK_PARALLELISM() { - return KEY_FLINK_PARALLELISM(null); - } - - public static String KEY_FLINK_PARALLELISM(String prefix) { - return (prefix != null ? prefix : "") + "parallelism.default"; - } - - public static String KEY_FLINK_OPTION_PREFIX() { - return "flink.option."; - } - - public static String KEY_FLINK_PROPERTY_PREFIX() { - return "flink.property."; - } - - public static String KEY_FLINK_TABLE_PREFIX() { - return "flink.table."; - } - - public static String KEY_SPARK_PROPERTY_PREFIX() { - return "spark."; - } - - public static String KEY_APP_PREFIX() { - return "app."; - } - - public static String KEY_SQL_PREFIX() { - return "sql."; - } - - public static String KEY_FLINK_APP_NAME() { - return "pipeline.name"; - } - - public static String KEY_YARN_APP_ID() { - return "yarn.application.id"; - } - - public static String KEY_YARN_APP_NAME() { - return "yarn.application.name"; - } - - public static String KEY_YARN_APP_QUEUE() { - return "yarn.application.queue"; - } - - public static String KEY_YARN_APP_NODE_LABEL() { - return "yarn.application.node-label"; - } - - public static String KEY_K8S_IMAGE_PULL_POLICY() { - return "kubernetes.container.image.pull-policy"; - } - - public static String FLINK_NATIVE_KUBERNETES_LABEL() { - return "flink-native-kubernetes"; - } - - /** about flink table */ - public static String KEY_FLINK_TABLE_PLANNER() { - return "flink.table.planner"; - } - - public static String KEY_FLINK_TABLE_MODE() { - return "flink.table.mode"; - } - - public static String KEY_FLINK_TABLE_CATALOG() { - return "flink.table.catalog"; - } - - public static String KEY_FLINK_TABLE_DATABASE() { - return "flink.table.database"; - } - - /** about config Kafka */ - public static String KAFKA_SINK_PREFIX() { - return "kafka.sink."; - } - - public static String KAFKA_SOURCE_PREFIX() { - return "kafka.source."; - } - - public static String KEY_KAFKA_TOPIC() { - return "topic"; - } - - public static String KEY_KAFKA_SEMANTIC() { - return "semantic"; - } - - public static String KEY_KAFKA_PATTERN() { - return "pattern"; - } - - public static String KEY_KAFKA_START_FROM() { - return "start.from"; - } - - public static String KEY_KAFKA_START_FROM_OFFSET() { - return "offset"; - } - - public static String KEY_KAFKA_START_FROM_TIMESTAMP() { - return "timestamp"; - } - - public static String KEY_ALIAS() { - return "alias"; - } - - /** about config jdbc... */ - public static String KEY_JDBC_PREFIX() { - return "jdbc."; - } - - public static String KEY_JDBC_DRIVER() { - return "driverClassName"; - } - - public static String KEY_JDBC_URL() { - return "jdbcUrl"; - } - - public static String KEY_JDBC_USER() { - return "username"; - } - - public static String KEY_JDBC_PASSWORD() { - return "password"; - } - - public static String KEY_JDBC_INSERT_BATCH() { - return "batch.size"; - } - - public static int DEFAULT_JDBC_INSERT_BATCH() { - return 1; - } - - public static String MONGO_PREFIX() { - return "mongodb."; - } - - /** about config HBase */ - public static String HBASE_PREFIX() { - return "hbase."; - } - - public static String KEY_HBASE_COMMIT_BATCH() { - return "hbase.commit.batch"; - } - - public static String KEY_HBASE_WRITE_SIZE() { - return "hbase.client.write.size"; - } - - public static int DEFAULT_HBASE_COMMIT_BATCH() { - return 1000; - } - - public static int DEFAULT_HBASE_WRITE_SIZE() { - return 1024 * 1024 * 10; - } - - public static String KEY_HBASE_AUTH_USER() { - return "hbase.auth.user"; - } - - /** about influx */ - public static String INFLUX_PREFIX() { - return "influx."; - } - - public static String KEY_FLINK_APPLICATION_MAIN_CLASS() { - return "$internal.application.main"; - } - - public static String KEY_FLINK_JM_PROCESS_MEMORY() { - return "jobmanager.memory.process.size"; - } - - public static String KEY_FLINK_TM_PROCESS_MEMORY() { - return "taskmanager.memory.process.size"; - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/conf/ConfigOption.java b/streampark-common/src/main/java/org/apache/streampark/common/conf/ConfigOption.java deleted file mode 100644 index f603e39157..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/conf/ConfigOption.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.conf; - -import org.apache.streampark.common.util.StringCastUtils; - -import java.util.Properties; -import java.util.function.Function; - -/** - * Configuration option with typed value retrieval. - * - * @param value type - */ -public class ConfigOption { - - private final String key; - private final T defaultValue; - private final boolean required; - private final Class classType; - private final String description; - private final Function handle; - private final String prefix; - private final Properties prop; - - public ConfigOption( - String key, - T defaultValue, - boolean required, - Class classType, - String description, - Function handle, - String prefix, - Properties prop) { - this.key = key; - this.defaultValue = defaultValue; - this.required = required; - this.classType = classType; - this.description = description; - this.handle = handle; - this.prefix = prefix != null ? prefix : ""; - this.prop = prop != null ? prop : new Properties(); - } - - public static Builder builder(String key) { - return new Builder<>(key); - } - - private String fullKey() { - if (prefix != null && !prefix.isEmpty()) { - return prefix + "." + key; - } - return key; - } - - @SuppressWarnings("unchecked") - public T get() { - String fk = fullKey(); - if (handle == null) { - if (required) { - Object v = prop.get(fk); - if (v == null) { - throw error("Is require"); - } - return StringCastUtils.cast(v.toString(), classType); - } else { - String v = prop.getProperty(fk); - if (v == null) { - return defaultValue; - } - return StringCastUtils.cast(v, classType); - } - } else { - try { - return handle.apply(fk); - } catch (Exception e) { - if (required) { - throw error(e.getMessage()); - } - return defaultValue; - } - } - } - - private IllegalArgumentException error(String message) { - return new IllegalArgumentException( - "[StreamPark] config error: key:" + fullKey() + ", detail: " + message); - } - - public static class Builder { - - private final String key; - private T defaultValue; - private boolean required; - private Class classType; - private String description = ""; - private Function handle; - private String prefix = ""; - private Properties prop = new Properties(); - - Builder(String key) { - this.key = key; - } - - public Builder defaultValue(T defaultValue) { - this.defaultValue = defaultValue; - return this; - } - - public Builder required(boolean required) { - this.required = required; - return this; - } - - public Builder classType(Class classType) { - this.classType = classType; - return this; - } - - public Builder description(String description) { - this.description = description; - return this; - } - - public Builder handle(Function handle) { - this.handle = handle; - return this; - } - - public Builder prefix(String prefix) { - this.prefix = prefix; - return this; - } - - public Builder properties(Properties prop) { - this.prop = prop; - return this; - } - - public ConfigOption build() { - return new ConfigOption<>(key, defaultValue, required, classType, description, handle, prefix, prop); - } - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/conf/InternalConfigHolder.java b/streampark-common/src/main/java/org/apache/streampark/common/conf/InternalConfigHolder.java deleted file mode 100644 index 32b7b2fb9e..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/conf/InternalConfigHolder.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.conf; - -import org.apache.streampark.common.constants.Constants; -import org.apache.streampark.common.util.StreamParkLoggerFactory; -import org.apache.streampark.common.util.StringCastUtils; -import org.apache.streampark.common.util.SystemPropertyUtils; - -import org.apache.streampark.shaded.org.slf4j.Logger; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; - -/** - * Thread-safe configuration storage containers. All configurations will be automatically - * initialized from the spring configuration items of the same name. - */ -public final class InternalConfigHolder { - - private static final Logger LOG = - StreamParkLoggerFactory.loggerFactory().getLogger(InternalConfigHolder.class.getName()); - - private static final int INITIAL_CAPACITY = 45; - - private static final Map CONF_DATA = new ConcurrentHashMap<>(INITIAL_CAPACITY); - private static final Map CONF_OPTIONS = - new ConcurrentHashMap<>(INITIAL_CAPACITY); - - static { - initConfigHub(); - } - - private InternalConfigHolder() { - } - - /** Initialize the ConfigHub. */ - public static void initConfigHub() { - CommonConfig.STREAMPARK_WORKSPACE_LOCAL(); - InternalOption unused = K8sFlinkConfig.jobStatusTrackTaskTimeoutSec; - } - - static void register(@Nonnull InternalOption conf) { - CONF_OPTIONS.put(conf.getKey(), conf); - if (conf.getDefaultValue() != null) { - CONF_DATA.put(conf.getKey(), conf.getDefaultValue()); - } - } - - @Nonnull - @SuppressWarnings("unchecked") - public static T get(@Nonnull InternalOption conf) { - Object value = CONF_DATA.get(conf.getKey()); - if (value == null || value == conf.getDefaultValue()) { - String v = SystemPropertyUtils.get(conf.getKey()); - if (v != null) { - if (!v.equals(value)) { - set(conf, v); - } - return StringCastUtils.cast(v, conf.getClassType()); - } else { - return (T) conf.getDefaultValue(); - } - } else { - return StringCastUtils.cast(value.toString(), conf.getClassType()); - } - } - - @Nonnull - @SuppressWarnings("unchecked") - public static T get(@Nonnull String key) { - Object v = CONF_DATA.get(key); - if (v == null) { - InternalOption conf = CONF_OPTIONS.get(key); - if (conf == null) { - InternalOption config = getConfig(key); - String sysVal = SystemPropertyUtils.get(key); - if (sysVal != null) { - return StringCastUtils.cast(sysVal, config.getClassType()); - } - throw new IllegalArgumentException("Config key has not been registered: " + key); - } - return (T) conf.getDefaultValue(); - } - return (T) v; - } - - @Nullable - public static InternalOption getConfig(String key) { - return CONF_OPTIONS.get(key); - } - - @Nonnull - public static Set keys() { - return CONF_OPTIONS.keySet(); - } - - public static void set(@Nonnull InternalOption conf, Object value) { - if (!CONF_OPTIONS.containsKey(conf.getKey())) { - throw new IllegalArgumentException("config key has not been registered: " + conf); - } - if (value == null) { - CONF_DATA.remove(conf.getKey()); - } else if (conf.getClassType() != value.getClass()) { - throw new IllegalArgumentException( - "config value type is not match of " - + conf.getKey() - + ", required: " - + conf.getClassType() - + ", actual: " - + value.getClass()); - } else { - SystemPropertyUtils.set(conf.getKey(), value.toString()); - CONF_DATA.put(conf.getKey(), value); - } - } - - public static void log() { - Set configKeys = keys(); - String details = - configKeys.stream() - .map( - key -> key - + " = " - + (key.contains("password") - ? Constants.DEFAULT_DATAMASK_STRING - : get(key))) - .collect(Collectors.joining("\n ")); - LOG.info( - "[StreamPark] Registered configs:\nConfigHub collected configs: {}\n {}", - configKeys.size(), - details); - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/conf/InternalOption.java b/streampark-common/src/main/java/org/apache/streampark/common/conf/InternalOption.java deleted file mode 100644 index 532e21fc36..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/conf/InternalOption.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.conf; - -/** Internal use of the system configuration option. */ -public class InternalOption { - - private final String key; - private final Object defaultValue; - private final Class classType; - private final String description; - - public InternalOption(String key, Object defaultValue, Class classType) { - this(key, defaultValue, classType, ""); - } - - public InternalOption(String key, Object defaultValue, Class classType, String description) { - this.key = key; - this.defaultValue = defaultValue; - this.classType = classType; - this.description = description; - InternalConfigHolder.register(this); - } - - public String getKey() { - return key; - } - - public Object getDefaultValue() { - return defaultValue; - } - - public Class getClassType() { - return classType; - } - - public String getDescription() { - return description; - } - - /** Scala API alias for {@link #getKey()}. */ - public String key() { - return getKey(); - } - - /** Scala API alias for {@link #getDefaultValue()}. */ - public Object defaultValue() { - return getDefaultValue(); - } - - /** Scala API alias for {@link #getClassType()}. */ - public Class classType() { - return getClassType(); - } - - @Override - public String toString() { - return "InternalOption{key='" + key + "'}"; - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/conf/K8sFlinkConfig.java b/streampark-common/src/main/java/org/apache/streampark/common/conf/K8sFlinkConfig.java deleted file mode 100644 index 1de257168c..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/conf/K8sFlinkConfig.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.conf; - -/** Flink kubernetes Configuration for v1 version */ -public final class K8sFlinkConfig { - - private K8sFlinkConfig() { - } - - @Deprecated(since = "3.0.0", forRemoval = true) - public static final InternalOption jobStatusTrackTaskTimeoutSec = - new InternalOption( - "streampark.flink-k8s.tracking.polling-task-timeout-sec.job-status", - 120L, - Long.class, - "run timeout seconds of single flink-k8s metrics tracking task"); - - @Deprecated(since = "3.0.0", forRemoval = true) - public static final InternalOption jobStatusTrackCacheTimeoutSec = - new InternalOption( - "streampark.flink-k8s.tracking.cache-timeout-sec.job-status", - 300, - Integer.class, - "status cache timeout seconds of single flink-k8s job status tracking task"); - - @Deprecated(since = "3.0.0", forRemoval = true) - public static final InternalOption metricTrackTaskTimeoutSec = - new InternalOption( - "streampark.flink-k8s.tracking.polling-task-timeout-sec.cluster-metric", - 120L, - Long.class, - "run timeout seconds of single flink-k8s job status tracking task"); - - @Deprecated(since = "3.0.0", forRemoval = true) - public static final InternalOption jobStatueTrackTaskIntervalSec = - new InternalOption( - "streampark.flink-k8s.tracking.polling-interval-sec.job-status", - 5L, - Long.class, - "interval seconds between two single flink-k8s metrics tracking task"); - - @Deprecated(since = "3.0.0", forRemoval = true) - public static final InternalOption metricTrackTaskIntervalSec = - new InternalOption( - "streampark.flink-k8s.tracking.polling-interval-sec.cluster-metric", - 5L, - Long.class, - "interval seconds between two single flink-k8s metrics tracking task"); - - @Deprecated(since = "3.0.0", forRemoval = true) - public static final InternalOption silentStateJobKeepTrackingSec = - new InternalOption( - "streampark.flink-k8s.tracking.silent-state-keep-sec", - 60, - Integer.class, - "retained tracking time for SILENT state flink tasks"); - - public static final InternalOption ingressClass = - new InternalOption( - "streampark.flink-k8s.ingress.class", - "nginx", - String.class, - "Direct ingress to the ingress controller."); - - @Deprecated(since = "3.0.0", forRemoval = true) - public static final String DEFAULT_KUBERNETES_NAMESPACE = "default"; -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/conf/Workspace.java b/streampark-common/src/main/java/org/apache/streampark/common/conf/Workspace.java deleted file mode 100644 index ea93ed5b70..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/conf/Workspace.java +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.conf; - -import org.apache.streampark.common.enums.StorageType; -import org.apache.streampark.common.util.HdfsUtils; -import org.apache.streampark.common.util.StringCastUtils; -import org.apache.streampark.common.util.SystemPropertyUtils; - -import java.net.URI; - -/** Workspace path configuration for local and remote storage. */ -public final class Workspace { - - private static final Workspace LOCAL = of(StorageType.LFS); - private static final Workspace REMOTE = of(StorageType.HDFS); - - /** Scala-friendly local workspace singleton. */ - public static final Workspace local = LOCAL; - - /** Scala-friendly remote workspace singleton. */ - public static final Workspace remote = REMOTE; - - static { - LOCAL.initPaths(); - } - - private final StorageType storageType; - - /** - * Path fields are private so Scala field-style access (e.g. {@code Workspace.local.APP_JARS}) - * resolves to accessor methods and triggers lazy initialization. Public fields would bypass - * {@link #ensurePathsInitialized()} and stay null when remote paths are deferred. - */ - private String WORKSPACE; - private String APP_PLUGINS; - private String APP_CLIENT; - private String APP_SHIMS; - private String APP_UPLOADS; - private String APP_PYTHON; - private String APP_PYTHON_VENV; - private String APP_WORKSPACE; - private String APP_FLINK; - private String APP_SPARK; - private String APP_BACKUPS; - private String APP_SAVEPOINTS; - private String APP_JARS; - - private volatile boolean pathsInitialized; - - private Workspace(StorageType storageType) { - this.storageType = storageType; - } - - public static Workspace of(StorageType storageType) { - return new Workspace(storageType); - } - - public static Workspace local() { - return LOCAL; - } - - public static Workspace remote() { - return REMOTE; - } - - /** local build path */ - public static String APP_LOCAL_DIST() { - return local.WORKSPACE() + "/dist"; - } - - /** dirPath of the maven local repository with built-in compilation process */ - public static String MAVEN_LOCAL_PATH() { - return local.WORKSPACE() + "/mvnrepo"; - } - - /** local sourceCode path.(for git...) */ - public static String PROJECT_LOCAL_PATH() { - return local.WORKSPACE() + "/project"; - } - - /** local log path. */ - public static String LOG_LOCAL_PATH() { - return local.WORKSPACE() + "/logs"; - } - - /** project build log path. */ - public static String PROJECT_BUILD_LOG_PATH() { - return LOG_LOCAL_PATH() + "/build_logs"; - } - - /** project archives path */ - public static String ARCHIVES_FILE_PATH() { - return remote.WORKSPACE() + "/historyserver/archive"; - } - - public String WORKSPACE() { - ensurePathsInitialized(); - return WORKSPACE; - } - - public String APP_PLUGINS() { - ensurePathsInitialized(); - return APP_PLUGINS; - } - - public String APP_CLIENT() { - ensurePathsInitialized(); - return APP_CLIENT; - } - - public String APP_SHIMS() { - ensurePathsInitialized(); - return APP_SHIMS; - } - - public String APP_UPLOADS() { - ensurePathsInitialized(); - return APP_UPLOADS; - } - - public String APP_PYTHON() { - ensurePathsInitialized(); - return APP_PYTHON; - } - - public String APP_PYTHON_VENV() { - ensurePathsInitialized(); - return APP_PYTHON_VENV; - } - - public String APP_WORKSPACE() { - ensurePathsInitialized(); - return APP_WORKSPACE; - } - - public String APP_FLINK() { - ensurePathsInitialized(); - return APP_FLINK; - } - - public String APP_SPARK() { - ensurePathsInitialized(); - return APP_SPARK; - } - - public String APP_BACKUPS() { - ensurePathsInitialized(); - return APP_BACKUPS; - } - - public String APP_SAVEPOINTS() { - ensurePathsInitialized(); - return APP_SAVEPOINTS; - } - - public String APP_JARS() { - ensurePathsInitialized(); - return APP_JARS; - } - - public String getAppJars() { - return APP_JARS(); - } - - private void ensurePathsInitialized() { - if (!pathsInitialized) { - synchronized (this) { - if (!pathsInitialized) { - initPaths(); - pathsInitialized = true; - } - } - } - } - - private void initPaths() { - if (pathsInitialized) { - return; - } - WORKSPACE = computeWorkspace(); - APP_PLUGINS = WORKSPACE + "/plugins"; - APP_CLIENT = WORKSPACE + "/client"; - APP_SHIMS = WORKSPACE + "/shims"; - APP_UPLOADS = WORKSPACE + "/uploads"; - APP_PYTHON = WORKSPACE + "/python"; - APP_PYTHON_VENV = APP_PYTHON + "/venv.zip"; - APP_WORKSPACE = WORKSPACE + "/workspace"; - APP_FLINK = WORKSPACE + "/flink"; - APP_SPARK = WORKSPACE + "/spark"; - APP_BACKUPS = WORKSPACE + "/backups"; - APP_SAVEPOINTS = WORKSPACE + "/savepoints"; - APP_JARS = WORKSPACE + "/jars"; - pathsInitialized = true; - } - - private String computeWorkspace() { - if (storageType == StorageType.LFS) { - String path = getConfigValue(CommonConfig.STREAMPARK_WORKSPACE_LOCAL()); - if (path == null) { - throw new IllegalArgumentException("[StreamPark] streampark.workspace.local must not be null"); - } - return path; - } - String path = getConfigValue(CommonConfig.STREAMPARK_WORKSPACE_REMOTE()); - if (path == null || path.isEmpty()) { - return HdfsUtils.getDefaultFS() + CommonConfig.STREAMPARK_WORKSPACE_REMOTE().getDefaultValue(); - } - String defaultFs = HdfsUtils.getDefaultFS(); - if (path.startsWith("hdfs://")) { - if (path.startsWith(defaultFs)) { - return path; - } - String hdfsPath = URI.create(path).getPath(); - return defaultFs + hdfsPath; - } - return defaultFs + path; - } - - @SuppressWarnings("unchecked") - private T getConfigValue(InternalOption option) { - String systemValue = SystemPropertyUtils.get(option.getKey()); - Object configValue = InternalConfigHolder.get(option); - Object defaultValue = option.getDefaultValue(); - if (systemValue == null && configValue == null) { - return (T) defaultValue; - } - if (systemValue == null) { - return (T) configValue; - } - if (configValue == null) { - return StringCastUtils.cast(systemValue, option.getClassType()); - } - if (configValue.equals(defaultValue)) { - return StringCastUtils.cast(systemValue, option.getClassType()); - } - return (T) configValue; - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineParser.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineParser.java new file mode 100644 index 0000000000..1854187cde --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineParser.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Strict parser for StreamPark's {@code --key value} and {@code --key=value} arguments. + * + *

This parser consumes an already-tokenized argument array, normally supplied by a JVM main + * method. It does not perform shell parsing; command strings must first pass through {@link + * CommandLineTokenizer}. Parsed values are assigned {@link ConfigSource#COMMAND_LINE} precedence. + */ +public final class CommandLineParser { + + /** Prefix identifying a StreamPark long option. */ + public static final String LONG_OPTION_PREFIX = "--"; + + private CommandLineParser() { + } + + /** + * Parses command-line arguments into a configuration snapshot. + * + *

A key without an explicit value is represented by boolean {@code true}. Positional + * arguments and duplicate keys are rejected because both otherwise hide deployment mistakes. + * + * @param arguments tokenized command-line arguments + * @return immutable command-line configuration snapshot + * @throws ConfigException when an argument is positional, malformed, duplicated, or null + */ + public static Configuration parse(String[] arguments) { + Objects.requireNonNull(arguments, "arguments must not be null"); + Map values = new LinkedHashMap<>(); + for (int index = 0; index < arguments.length; index++) { + String token = Objects.requireNonNull(arguments[index], "argument must not be null"); + if (!token.startsWith(LONG_OPTION_PREFIX) || token.length() == LONG_OPTION_PREFIX.length()) { + throw new ConfigException("Expected a --key argument but found '" + token + "'"); + } + + String expression = token.substring(LONG_OPTION_PREFIX.length()); + int separator = expression.indexOf('='); + String key; + Object value; + // An equals sign binds even an empty value to the current option. Without one, the + // following non-option token is consumed; otherwise the option is treated as a flag. + if (separator >= 0) { + key = expression.substring(0, separator); + value = expression.substring(separator + 1); + } else { + key = expression; + if (index + 1 < arguments.length + && !arguments[index + 1].startsWith(LONG_OPTION_PREFIX)) { + value = arguments[++index]; + } else { + value = true; + } + } + if (key.trim().isEmpty()) { + throw new ConfigException("Command-line option key must not be blank"); + } + if (values.putIfAbsent(key, value) != null) { + throw new ConfigException("Duplicate command-line option: " + key); + } + } + return Configuration.builder() + .add("command line", ConfigSource.COMMAND_LINE, values) + .build(); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineTokenizer.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineTokenizer.java new file mode 100644 index 0000000000..a88046ce97 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineTokenizer.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Tokenizes a command-line string without invoking a shell or performing variable expansion. + * + *

The result is suitable for parsers that consume a JVM-style {@code String[]} argument array. + * Avoiding a shell is a security boundary: metacharacters, substitutions, and redirects remain + * ordinary token content and are never executed. + */ +public final class CommandLineTokenizer { + + private CommandLineTokenizer() { + } + + /** + * Splits a command line while honoring single quotes, double quotes, and backslash escapes. + * + *

The tokenizer intentionally implements only argument grouping. Shell substitutions, + * redirects, globbing, and environment expansion are never evaluated. + * + * @param commandLine command string; {@code null} and blank input produce an empty list + * @return immutable tokens with grouping quotes and escape characters removed + * @throws ConfigException when a quote or escape sequence is incomplete + */ + public static List tokenize(String commandLine) { + if (commandLine == null || commandLine.trim().isEmpty()) { + return Collections.emptyList(); + } + + List tokens = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + char quote = 0; + boolean escaping = false; + boolean tokenStarted = false; + // A single pass is sufficient because the tokenizer tracks only quote and escape state. + // tokenStarted preserves deliberately empty quoted arguments such as "". + for (int index = 0; index < commandLine.length(); index++) { + char character = commandLine.charAt(index); + if (escaping) { + current.append(character); + escaping = false; + tokenStarted = true; + continue; + } + if (character == '\\' && quote != '\'') { + escaping = true; + tokenStarted = true; + continue; + } + if (quote != 0) { + if (character == quote) { + quote = 0; + } else { + current.append(character); + } + tokenStarted = true; + continue; + } + if (character == '\'' || character == '"') { + quote = character; + tokenStarted = true; + } else if (Character.isWhitespace(character)) { + if (tokenStarted) { + tokens.add(current.toString()); + current.setLength(0); + tokenStarted = false; + } + } else { + current.append(character); + tokenStarted = true; + } + } + if (escaping) { + throw new ConfigException("Command line ends with an incomplete escape sequence"); + } + if (quote != 0) { + throw new ConfigException("Command line contains an unterminated quoted value"); + } + if (tokenStarted) { + tokens.add(current.toString()); + } + return Collections.unmodifiableList(tokens); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigException.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigException.java new file mode 100644 index 0000000000..4714d090b1 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigException.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +/** + * Indicates invalid, missing, or unreadable configuration. + * + *

This unchecked exception is the common failure boundary for parsing, option conversion, + * validation, and source composition. Callers can therefore report configuration failures without + * depending on parser-specific exception types. + */ +public class ConfigException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + /** + * Creates a configuration exception with an operator-facing diagnostic. + * + * @param message diagnostic that identifies the invalid option, source, or input + */ + public ConfigException(String message) { + super(message); + } + + /** + * Creates a configuration exception retaining the underlying parser or conversion failure. + * + * @param message diagnostic that identifies the invalid option, source, or input + * @param cause underlying failure + */ + public ConfigException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOption.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOption.java new file mode 100644 index 0000000000..3220d3b9ef --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOption.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.io.Serializable; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; + +/** + * Immutable metadata for a typed configuration value. + * + *

An option never reads global state. Values are resolved by a {@link ReadableConfig}, which + * keeps option declaration separate from configuration loading and precedence. + * + * @param value type + */ +public final class ConfigOption implements Serializable { + + private static final long serialVersionUID = 1L; + + private final String key; + private final ConfigValueType valueType; + private final T defaultValue; + private final boolean hasDefaultValue; + private final String description; + private final List fallbackKeys; + private final boolean sensitive; + private final Predicate validator; + private final String validationMessage; + + public ConfigOption(String key, + ConfigValueType valueType, + T defaultValue, + boolean hasDefaultValue, + String description, + List fallbackKeys, + boolean sensitive, + Predicate validator, + String validationMessage) { + this.key = requireKey(key); + this.valueType = Objects.requireNonNull(valueType, "valueType must not be null"); + this.hasDefaultValue = hasDefaultValue; + this.description = Objects.requireNonNull(description, "description must not be null"); + this.fallbackKeys = List.copyOf(Objects.requireNonNull(fallbackKeys, "fallbackKeys must not be null")); + this.sensitive = sensitive; + this.validator = Objects.requireNonNull(validator, "validator must not be null"); + this.validationMessage = + Objects.requireNonNull(validationMessage, "validationMessage must not be null"); + if (hasDefaultValue) { + this.defaultValue = Configuration.immutableValue(defaultValue); + validate(this.defaultValue); + } else { + this.defaultValue = null; + } + } + + /** + * Returns the canonical configuration key. + * + * @return canonical key + */ + public String key() { + return key; + } + + /** + * Returns whether this option declares a default value. + * + * @return whether a default exists + */ + public boolean hasDefaultValue() { + return hasDefaultValue; + } + + /** + * Returns the declared default value. + * + * @return declared default + * @throws ConfigException when the option has no default + */ + public T defaultValue() { + if (!hasDefaultValue) { + throw new ConfigException("Configuration option '" + key + "' has no default value"); + } + return defaultValue; + } + + /** + * Returns the human-readable option description. + * + * @return option description + */ + public String description() { + return description; + } + + /** + * Returns legacy keys considered after the canonical key. + * + * @return immutable fallback-key list in resolution order + */ + public List fallbackKeys() { + return fallbackKeys; + } + + /** + * Returns whether conversion errors for this option must redact the supplied value. + * + * @return whether this option contains sensitive material + */ + public boolean sensitive() { + return sensitive; + } + + Class valueClass() { + return valueType.valueClass(); + } + + T convert(Object value) { + T converted = valueType.convert(value); + validate(converted); + return converted; + } + + String format(T value) { + validate(value); + return valueType.format(value); + } + + void validate(T value) { + if (value == null) { + throw new ConfigException("Configuration option '" + key + "' must not be null"); + } + if (!validator.test(value)) { + throw new ConfigException( + "Invalid value for configuration option '" + key + "': " + validationMessage); + } + } + + private static String requireKey(String key) { + String normalized = Objects.requireNonNull(key, "key must not be null").trim(); + if (normalized.isEmpty()) { + throw new IllegalArgumentException("key must not be blank"); + } + return normalized; + } + + @Override + public boolean equals(Object object) { + return object instanceof ConfigOption && key.equals(((ConfigOption) object).key); + } + + @Override + public int hashCode() { + return key.hashCode(); + } + + @Override + public String toString() { + return "ConfigOption{" + "key='" + key + '\'' + ", type=" + valueClass().getSimpleName() + '}'; + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOptions.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOptions.java new file mode 100644 index 0000000000..a48880dea2 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOptions.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.function.Predicate; + +/** + * Entry point for staged, type-safe {@link ConfigOption} declarations. + * + *

The staged API requires an option type before defaults, validation, fallback keys, and + * documentation can be declared. Calling {@code build()} produces immutable metadata and performs + * validation of the declared default. + */ +public final class ConfigOptions { + + private ConfigOptions() { + } + + /** + * Starts an option declaration for a canonical key. + * + * @param key non-blank canonical configuration key + * @return builder that requires a value type before option metadata can be completed + */ + public static OptionBuilder key(String key) { + return new OptionBuilder(key); + } + + /** + * First stage of an option declaration, used to select its value type. + * + *

Separating this stage prevents defaults and validators from being declared before Java + * knows the option's value type. + */ + public static final class OptionBuilder { + + private final String key; + + private OptionBuilder(String key) { + this.key = key; + } + + /** + * Declares a string option. + * + * @return typed metadata builder + */ + public TypedOptionBuilder stringType() { + return typed(ConfigValueTypes.STRING); + } + + /** + * Declares a 32-bit integer option. + * + * @return typed metadata builder + */ + public TypedOptionBuilder intType() { + return typed(ConfigValueTypes.INTEGER); + } + + /** + * Declares a 64-bit integer option. + * + * @return typed metadata builder + */ + public TypedOptionBuilder longType() { + return typed(ConfigValueTypes.LONG); + } + + /** + * Declares a strict boolean option. + * + *

Only boolean values and case-insensitive {@code true}/{@code false} strings are + * accepted. + * + * @return typed metadata builder + */ + public TypedOptionBuilder booleanType() { + return typed(ConfigValueTypes.BOOLEAN); + } + + /** + * Declares a duration option whose bare numbers represent milliseconds. + * + * @return typed metadata builder + */ + public TypedOptionBuilder durationType() { + return durationType(ChronoUnit.MILLIS); + } + + /** + * Declares a duration option with an explicit unit for bare numeric values. + * + * @param unitForBareNumbers unit assigned to values without a suffix + * @return typed metadata builder + */ + public TypedOptionBuilder durationType(ChronoUnit unitForBareNumbers) { + return typed(ConfigValueTypes.duration(unitForBareNumbers)); + } + + /** + * Declares a binary data-size option. + * + * @return typed metadata builder + */ + public TypedOptionBuilder dataSizeType() { + return typed(ConfigValueTypes.DATA_SIZE); + } + + /** + * Declares a case-insensitive enum option. + * + * @param enumClass enum class defining accepted constants + * @param enum value type + * @return typed metadata builder + */ + public > TypedOptionBuilder enumType(Class enumClass) { + return typed(ConfigValueTypes.enumType(enumClass)); + } + + /** + * Declares a comma-separated or structured string-list option. + * + * @return typed metadata builder + */ + public TypedOptionBuilder> stringListType() { + return typed(ConfigValueTypes.STRING_LIST); + } + + private TypedOptionBuilder typed(ConfigValueType valueType) { + return new TypedOptionBuilder<>(key, valueType); + } + } + + /** + * Second stage of an option declaration, used to complete immutable metadata. + * + *

Builder instances are mutable and intended for a single declaration. The resulting + * {@link ConfigOption} defensively copies fallback keys and validates its default value. + * + * @param option value type + */ + public static final class TypedOptionBuilder { + + private final String key; + private final ConfigValueType valueType; + private T defaultValue; + private boolean hasDefaultValue; + private String description = ""; + private List fallbackKeys = Collections.emptyList(); + private boolean sensitive; + private Predicate validator = ignored -> true; + private String validationMessage = "value does not satisfy the option constraint"; + + private TypedOptionBuilder(String key, ConfigValueType valueType) { + this.key = key; + this.valueType = valueType; + } + + /** + * Declares a default value. + * + *

The value is validated when {@link #build()} constructs the option. + * + * @param value non-null default value + * @return this builder + */ + public TypedOptionBuilder defaultValue(T value) { + this.defaultValue = Objects.requireNonNull(value, "default value must not be null"); + this.hasDefaultValue = true; + return this; + } + + /** + * Declares that callers must explicitly provide the option. + * + * @return this builder + */ + public TypedOptionBuilder noDefaultValue() { + this.defaultValue = null; + this.hasDefaultValue = false; + return this; + } + + /** + * Sets the public option description. + * + * @param description human-readable description + * @return this builder + */ + public TypedOptionBuilder withDescription(String description) { + this.description = Objects.requireNonNull(description, "description must not be null"); + return this; + } + + /** + * Adds legacy fallback keys in resolution order. + * + *

The canonical key always wins. Fallbacks provide a migration path and do not create + * aliases in an effective configuration snapshot. + * + * @param fallbackKeys legacy keys considered after the canonical key + * @return this builder + */ + public TypedOptionBuilder withFallbackKeys(String... fallbackKeys) { + this.fallbackKeys = Arrays.asList(fallbackKeys.clone()); + return this; + } + + /** + * Marks conversion failures as sensitive so their raw value is redacted. + * + * @return this builder + */ + public TypedOptionBuilder sensitive() { + this.sensitive = true; + return this; + } + + /** + * Adds a semantic value constraint evaluated after type conversion. + * + * @param validator predicate that accepts valid converted values + * @param validationMessage diagnostic used when the predicate rejects a value + * @return this builder + */ + public TypedOptionBuilder check(Predicate validator, String validationMessage) { + this.validator = Objects.requireNonNull(validator, "validator must not be null"); + this.validationMessage = + Objects.requireNonNull(validationMessage, "validationMessage must not be null"); + return this; + } + + /** + * Builds immutable option metadata and validates its declared default. + * + * @return completed option metadata + * @throws ConfigException when the declared default violates the option constraint + * @throws IllegalArgumentException when a fallback key is blank + */ + public ConfigOption build() { + List keys = new ArrayList<>(fallbackKeys.size()); + for (String fallbackKey : fallbackKeys) { + String keyValue = Objects.requireNonNull(fallbackKey, "fallback key must not be null").trim(); + if (keyValue.isEmpty()) { + throw new IllegalArgumentException("fallback key must not be blank"); + } + keys.add(keyValue); + } + return new ConfigOption<>( + key, + valueType, + defaultValue, + hasDefaultValue, + description, + keys, + sensitive, + validator, + validationMessage); + } + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOrigin.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOrigin.java new file mode 100644 index 0000000000..41542319f8 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOrigin.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.io.Serializable; +import java.util.Objects; + +/** + * Immutable metadata describing where an effective configuration value originated. + * + *

The source category participates in precedence decisions, while the human-readable name is + * retained for diagnostics. Origins describe values rather than entire snapshots because a + * snapshot can combine entries from multiple files and runtime layers. + */ +public final class ConfigOrigin implements Serializable { + + private static final long serialVersionUID = 1L; + + private final ConfigSource source; + private final String name; + + private ConfigOrigin(ConfigSource source, String name) { + this.source = source; + this.name = name; + } + + /** + * Creates validated source metadata with a diagnostic name. + * + * @param source source category used for precedence + * @param name non-blank source name, such as a file path or {@code command line} + * @return immutable source metadata + */ + public static ConfigOrigin of(ConfigSource source, String name) { + Objects.requireNonNull(source, "source must not be null"); + String normalized = Objects.requireNonNull(name, "origin name must not be null").trim(); + if (normalized.isEmpty()) { + throw new IllegalArgumentException("origin name must not be blank"); + } + return new ConfigOrigin(source, normalized); + } + + /** + * Returns the source category used for precedence. + * + * @return source category + */ + public ConfigSource source() { + return source; + } + + /** + * Returns the human-readable source name used in diagnostics. + * + * @return non-blank source name + */ + public String name() { + return name; + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof ConfigOrigin)) { + return false; + } + ConfigOrigin that = (ConfigOrigin) object; + return source == that.source && name.equals(that.name); + } + + @Override + public int hashCode() { + return Objects.hash(source, name); + } + + @Override + public String toString() { + return source + ":" + name; + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigPrefix.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigPrefix.java new file mode 100644 index 0000000000..64ee390d42 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigPrefix.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.io.Serializable; +import java.util.Objects; + +/** + * A validated configuration namespace prefix. + * + *

Using a value object instead of scattered string constants makes prefix removal explicit and + * prevents accidental substring operations on unrelated keys. + */ +public final class ConfigPrefix implements Serializable { + + private static final long serialVersionUID = 1L; + + private final String value; + + private ConfigPrefix(String value) { + this.value = value; + } + + /** + * Creates a non-blank prefix value. + * + * @param value literal namespace prefix + * @return validated prefix + */ + public static ConfigPrefix of(String value) { + String normalized = Objects.requireNonNull(value, "prefix must not be null").trim(); + if (normalized.isEmpty()) { + throw new IllegalArgumentException("prefix must not be blank"); + } + return new ConfigPrefix(normalized); + } + + /** + * Returns the literal prefix. + * + * @return prefix text + */ + public String value() { + return value; + } + + /** + * Returns whether the supplied key belongs to this namespace. + * + * @param key key to inspect; {@code null} never matches + * @return whether the key starts with this prefix + */ + public boolean matches(String key) { + return key != null && key.startsWith(value); + } + + /** + * Removes this prefix from a configuration key. + * + * @param key key belonging to this namespace + * @return suffix after the prefix + * @throws ConfigException when the key does not start with this prefix + */ + public String stripFrom(String key) { + if (!matches(key)) { + throw new ConfigException("Configuration key '" + key + "' does not start with '" + value + "'"); + } + return key.substring(value.length()); + } + + @Override + public boolean equals(Object object) { + return object instanceof ConfigPrefix && value.equals(((ConfigPrefix) object).value); + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public String toString() { + return value; + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigSource.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigSource.java new file mode 100644 index 0000000000..d95adf3f25 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigSource.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +/** + * Configuration sources in ascending precedence order. + * + *

The declaration order is part of the merge contract used by {@link Configuration.Builder}. + * New sources must be placed according to their intended precedence; reordering existing constants + * changes which value becomes effective. + */ +public enum ConfigSource { + /** Built-in values supplied before external configuration is loaded. */ + DEFAULTS, + + /** Values parsed from configuration files. */ + FILE, + + /** Values obtained from process environment variables. */ + ENVIRONMENT, + + /** Values obtained from JVM system properties. */ + SYSTEM_PROPERTIES, + + /** Values supplied as application or launcher arguments. */ + COMMAND_LINE, + + /** Values set explicitly by a running StreamPark component. */ + RUNTIME +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/enums/ApiType.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigValueType.java similarity index 51% rename from streampark-common/src/main/java/org/apache/streampark/common/enums/ApiType.java rename to streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigValueType.java index c03ec2b6c2..51b1794ad8 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/enums/ApiType.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigValueType.java @@ -15,24 +15,25 @@ * limitations under the License. */ -package org.apache.streampark.common.enums; +package org.apache.streampark.common.configuration; -import javax.annotation.Nonnull; +/** + * Internal conversion and canonical-format contract for one option value type. + * + *

Implementations must be stateless and thread-safe because {@link ConfigOption} instances are + * shared constants. Conversion accepts the raw representation retained by {@link Configuration}; + * formatting produces the stable representation exposed by untyped views. + * + * @param converted value type + */ +interface ConfigValueType { -/** Flink application API type. */ -public enum ApiType { + /** Returns the runtime class of converted values. */ + Class valueClass(); - JAVA, - SCALA; + /** Converts a non-null raw configuration value. */ + T convert(Object value); - /** - * Resolve enum constant by name (Scala {@code Enumeration.withName} compatibility). - * - * @param name enum constant name - * @return matching {@link ApiType} - */ - @Nonnull - public static ApiType withName(@Nonnull String name) { - return valueOf(name); - } + /** Formats a converted value for an untyped configuration view. */ + String format(T value); } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigValueTypes.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigValueTypes.java new file mode 100644 index 0000000000..272cd8569c --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigValueTypes.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.time.Duration; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Built-in strict converters used by {@link ConfigOptions}. + * + *

Converters accept values produced by YAML, HOCON, properties, and command-line parsers while + * rejecting ambiguous coercions. In particular, booleans accept only {@code true}/{@code false}, + * integral types reject fractional numbers, and durations reject negative values. Keeping these + * rules in one package-private catalog makes option conversion consistent across modules. + */ +final class ConfigValueTypes { + + private static final Pattern DURATION_PATTERN = Pattern.compile("^([+-]?\\d+)\\s*([a-zA-Z]*)$"); + + static final ConfigValueType STRING = + simple(String.class, value -> value instanceof String ? (String) value : value.toString(), value -> value); + + static final ConfigValueType INTEGER = + simple(Integer.class, ConfigValueTypes::toInteger, Object::toString); + + static final ConfigValueType LONG = + simple(Long.class, ConfigValueTypes::toLong, Object::toString); + + static final ConfigValueType BOOLEAN = + simple(Boolean.class, ConfigValueTypes::toBoolean, Object::toString); + + static final ConfigValueType DATA_SIZE = + simple( + DataSize.class, + value -> value instanceof DataSize ? (DataSize) value : DataSize.parse(value.toString()), + DataSize::toString); + + @SuppressWarnings("unchecked") + static final ConfigValueType> STRING_LIST = + simple( + (Class>) (Class) List.class, + ConfigValueTypes::toStringList, + values -> String.join(",", values)); + + private ConfigValueTypes() { + } + + static ConfigValueType duration(ChronoUnit unitForBareNumbers) { + Objects.requireNonNull(unitForBareNumbers, "duration unit must not be null"); + return simple( + Duration.class, + value -> toDuration(value, unitForBareNumbers), + Duration::toString); + } + + static > ConfigValueType enumType(Class enumClass) { + Objects.requireNonNull(enumClass, "enum class must not be null"); + return simple( + enumClass, + value -> { + if (enumClass.isInstance(value)) { + return enumClass.cast(value); + } + String text = value.toString().trim(); + for (E constant : enumClass.getEnumConstants()) { + if (constant.name().equalsIgnoreCase(text)) { + return constant; + } + } + throw new IllegalArgumentException( + "expected one of " + java.util.Arrays.toString(enumClass.getEnumConstants())); + }, + Enum::name); + } + + static String formatUnknown(Object value) { + if (value instanceof Collection) { + List values = new ArrayList<>(); + for (Object element : (Collection) value) { + values.add(String.valueOf(element)); + } + return String.join(",", values); + } + return String.valueOf(value); + } + + private static Integer toInteger(Object value) { + if (value instanceof Integer) { + return (Integer) value; + } + if (value instanceof Number) { + return Math.toIntExact(toLong(value)); + } + return Integer.valueOf(value.toString().trim()); + } + + private static Long toLong(Object value) { + if (value instanceof Long) { + return (Long) value; + } + if (value instanceof Number) { + Number number = (Number) value; + long longValue = number.longValue(); + // Number#longValue silently truncates floating-point input. Comparing the round-trip + // value rejects that lossy coercion before it reaches an integral option. + if (Double.compare(number.doubleValue(), (double) longValue) != 0) { + throw new IllegalArgumentException("expected an integral number"); + } + return longValue; + } + return Long.valueOf(value.toString().trim()); + } + + private static Boolean toBoolean(Object value) { + if (value instanceof Boolean) { + return (Boolean) value; + } + String text = value.toString().trim(); + if ("true".equalsIgnoreCase(text)) { + return true; + } + if ("false".equalsIgnoreCase(text)) { + return false; + } + throw new IllegalArgumentException("expected 'true' or 'false'"); + } + + private static Duration toDuration(Object value, ChronoUnit unitForBareNumbers) { + if (value instanceof Duration) { + return requireNonNegative((Duration) value); + } + if (value instanceof Number) { + return durationOf(toLong(value), unitForBareNumbers); + } + String text = value.toString().trim(); + try { + // ISO-8601 remains the canonical representation; compact suffixes support existing + // StreamPark configuration files and command-line values. + return requireNonNegative(Duration.parse(text)); + } catch (DateTimeParseException ignored) { + Matcher matcher = DURATION_PATTERN.matcher(text); + if (!matcher.matches()) { + throw new IllegalArgumentException("invalid duration: " + value); + } + long amount = Long.parseLong(matcher.group(1)); + String suffix = matcher.group(2).toLowerCase(Locale.ROOT); + ChronoUnit unit; + switch (suffix) { + case "": + unit = unitForBareNumbers; + break; + case "ns": + unit = ChronoUnit.NANOS; + break; + case "us": + unit = ChronoUnit.MICROS; + break; + case "ms": + unit = ChronoUnit.MILLIS; + break; + case "s": + unit = ChronoUnit.SECONDS; + break; + case "m": + case "min": + unit = ChronoUnit.MINUTES; + break; + case "h": + unit = ChronoUnit.HOURS; + break; + case "d": + unit = ChronoUnit.DAYS; + break; + default: + throw new IllegalArgumentException("unsupported duration unit: " + suffix); + } + return durationOf(amount, unit); + } + } + + private static Duration durationOf(long value, ChronoUnit unit) { + return requireNonNegative(Duration.of(value, unit)); + } + + private static Duration requireNonNegative(Duration duration) { + if (duration.isNegative()) { + throw new IllegalArgumentException("duration must not be negative"); + } + return duration; + } + + private static List toStringList(Object value) { + if (value instanceof Collection) { + List result = new ArrayList<>(); + for (Object element : (Collection) value) { + result.add(String.valueOf(element)); + } + return Collections.unmodifiableList(result); + } + String text = value.toString().trim(); + if (text.isEmpty()) { + return Collections.emptyList(); + } + String[] parts = text.split(",", -1); + List result = new ArrayList<>(parts.length); + for (String part : parts) { + result.add(part.trim()); + } + return Collections.unmodifiableList(result); + } + + private static ConfigValueType simple( + Class valueClass, + Function converter, + Function formatter) { + return new ConfigValueType() { + + @Override + public Class valueClass() { + return valueClass; + } + + @Override + public T convert(Object value) { + return converter.apply(Objects.requireNonNull(value, "configuration value must not be null")); + } + + @Override + public String format(T value) { + return formatter.apply(value); + } + }; + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java new file mode 100644 index 0000000000..587c71e17a --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java @@ -0,0 +1,416 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.io.Serializable; +import java.lang.reflect.Array; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Immutable configuration snapshot with typed access and value origins. + * + *

Each key retains only its effective value. Source precedence is enforced while a snapshot is + * built, so the result does not depend on the order in which different source types are added. + * Within the same source type, the value added last wins. Conversion is deliberately deferred + * until an option is read, which keeps parsing independent from the set of options available to a + * particular module. + */ +public final class Configuration implements ReadableConfig, Serializable { + + private static final long serialVersionUID = 1L; + private static final Configuration EMPTY = new Configuration(Collections.emptyMap()); + + private final Map values; + + private Configuration(Map values) { + this.values = Collections.unmodifiableMap(new LinkedHashMap<>(values)); + } + + /** + * Returns the shared empty configuration snapshot. + * + * @return immutable empty configuration + */ + public static Configuration empty() { + return EMPTY; + } + + /** + * Creates an empty snapshot builder. + * + * @return new configuration builder + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Creates a builder initialized with every value and origin from a base snapshot. + * + * @param base immutable configuration to copy + * @return new configuration builder + */ + public static Builder builder(Configuration base) { + return new Builder(base); + } + + @Override + public T get(ConfigOption option) { + Optional value = getOptional(option); + if (value.isPresent()) { + return value.get(); + } + if (option.hasDefaultValue()) { + return option.defaultValue(); + } + throw new ConfigException("Required configuration option '" + option.key() + "' is missing"); + } + + @Override + public Optional getOptional(ConfigOption option) { + Objects.requireNonNull(option, "option must not be null"); + // The canonical key always wins over fallbacks. Fallbacks are declaration-time migration + // metadata and never copied into the effective snapshot. + String resolvedKey = resolveKey(option); + if (resolvedKey == null) { + return Optional.empty(); + } + Value value = values.get(resolvedKey); + try { + return Optional.of(option.convert(value.rawValue)); + } catch (RuntimeException e) { + String rendered = + option.sensitive() || SensitiveKeys.isSensitive(resolvedKey) + ? SensitiveKeys.MASK + : ConfigValueTypes.formatUnknown(value.rawValue); + throw new ConfigException( + "Invalid value '" + + rendered + + "' for configuration option '" + + option.key() + + "' from " + + value.origin, + e); + } + } + + @Override + public boolean contains(ConfigOption option) { + return resolveKey(Objects.requireNonNull(option, "option must not be null")) != null; + } + + /** + * Returns whether the snapshot contains an exact raw key. + * + *

This method does not inspect option fallback keys. + * + * @param key exact configuration key + * @return whether the key is present + */ + public boolean containsKey(String key) { + return values.containsKey(Objects.requireNonNull(key, "key must not be null")); + } + + /** + * Returns an exact key as a string, without applying an option default or fallback. + * + * @param key exact configuration key + * @return canonical string representation of the raw value + * @throws ConfigException when the key is absent + */ + public String getString(String key) { + return getOptionalString(key) + .orElseThrow(() -> new ConfigException("Required configuration key '" + key + "' is missing")); + } + + /** + * Returns an exact key as a string when present. + * + * @param key exact configuration key + * @return canonical string representation, or an empty optional when absent + */ + public Optional getOptionalString(String key) { + Value value = values.get(Objects.requireNonNull(key, "key must not be null")); + return value == null + ? Optional.empty() + : Optional.of(ConfigValueTypes.formatUnknown(value.rawValue)); + } + + /** + * Returns source metadata retained for the effective value of an exact key. + * + * @param key exact configuration key + * @return effective value origin, or an empty optional when absent + */ + public Optional origin(String key) { + Value value = values.get(Objects.requireNonNull(key, "key must not be null")); + return value == null ? Optional.empty() : Optional.of(value.origin); + } + + /** + * Returns entries below a namespace with the prefix removed from each key. + * + *

The returned snapshot retains the origin of each value. Prefix matching is literal so + * callers should include the desired separator, for example {@code flink.option.}. + * + * @param prefix non-blank literal prefix + * @return immutable configuration containing the matching entries + */ + public Configuration subset(String prefix) { + String normalized = requirePrefix(prefix); + Map subset = new LinkedHashMap<>(); + values.forEach( + (key, value) -> { + if (key.startsWith(normalized)) { + subset.put(key.substring(normalized.length()), value); + } + }); + return subset.isEmpty() ? EMPTY : new Configuration(subset); + } + + /** + * Returns an immutable string representation of all effective entries. + * + *

This view is not redacted and must not be written to logs. Use {@link #toRedactedMap()} for + * diagnostics. + * + * @return unredacted effective entries + */ + public Map toMap() { + Map result = new LinkedHashMap<>(); + values.forEach((key, value) -> result.put(key, ConfigValueTypes.formatUnknown(value.rawValue))); + return Collections.unmodifiableMap(result); + } + + /** + * Returns effective entries with values whose key looks sensitive masked. + * + * @return immutable diagnostic view + */ + public Map toRedactedMap() { + Map result = new LinkedHashMap<>(); + values.forEach( + (key, value) -> result.put( + key, + SensitiveKeys.isSensitive(key) + ? SensitiveKeys.MASK + : ConfigValueTypes.formatUnknown(value.rawValue))); + return Collections.unmodifiableMap(result); + } + + /** + * Returns whether the snapshot contains no entries. + * + * @return whether no effective values exist + */ + public boolean isEmpty() { + return values.isEmpty(); + } + + /** + * Returns the number of effective entries. + * + * @return effective entry count + */ + public int size() { + return values.size(); + } + + private String resolveKey(ConfigOption option) { + if (values.containsKey(option.key())) { + return option.key(); + } + for (String fallbackKey : option.fallbackKeys()) { + if (values.containsKey(fallbackKey)) { + return fallbackKey; + } + } + return null; + } + + private static String requirePrefix(String prefix) { + String normalized = Objects.requireNonNull(prefix, "prefix must not be null").trim(); + if (normalized.isEmpty()) { + throw new IllegalArgumentException("prefix must not be blank"); + } + return normalized; + } + + /** + * Copies structured values into representations that cannot be changed through caller-owned + * collections. Scalar configuration types are immutable and can be retained directly. + */ + @SuppressWarnings("unchecked") + static T immutableValue(T value) { + Objects.requireNonNull(value, "configuration value must not be null"); + if (value instanceof Map) { + Map copy = new LinkedHashMap<>(); + ((Map) value).forEach( + (key, item) -> copy.put(immutableValue(key), immutableValue(item))); + return (T) Collections.unmodifiableMap(copy); + } + if (value instanceof Collection) { + List copy = new ArrayList<>(((Collection) value).size()); + for (Object item : (Collection) value) { + copy.add(immutableValue(item)); + } + return (T) Collections.unmodifiableList(copy); + } + if (value.getClass().isArray()) { + int length = Array.getLength(value); + Object copy = Array.newInstance(value.getClass().getComponentType(), length); + for (int index = 0; index < length; index++) { + Array.set(copy, index, immutableValue(Array.get(value, index))); + } + return (T) copy; + } + return value; + } + + @Override + public String toString() { + return toRedactedMap().toString(); + } + + private static final class Value implements Serializable { + + private static final long serialVersionUID = 1L; + + private final Object rawValue; + private final ConfigOrigin origin; + + private Value(Object rawValue, ConfigOrigin origin) { + this.rawValue = immutableValue(rawValue); + this.origin = origin; + } + } + + /** + * Builds an immutable configuration snapshot. + * + *

Higher {@link ConfigSource} values replace lower ones regardless of insertion order. A + * later value replaces an earlier value only when both have the same source precedence. + */ + public static final class Builder { + + private final Map values = new LinkedHashMap<>(); + + private Builder() { + } + + private Builder(Configuration base) { + values.putAll(Objects.requireNonNull(base, "base configuration must not be null").values); + } + + /** + * Adds one named layer using the declared source precedence. + * + *

Entries are consumed immediately. Later layers of the same source replace earlier + * layers, while lower-precedence sources cannot replace higher-precedence values. + * + * @param originName non-blank name retained in diagnostics + * @param source source category used for precedence + * @param entries raw configuration entries + * @return this builder + */ + public Builder add(String originName, ConfigSource source, Map entries) { + ConfigOrigin origin = ConfigOrigin.of(source, originName); + Objects.requireNonNull(entries, "configuration entries must not be null") + .forEach((key, value) -> putRaw(key, value, origin)); + return this; + } + + /** + * Merges another snapshot while retaining each entry's original source metadata. + * + * @param configuration immutable snapshot to merge + * @return this builder + */ + public Builder add(Configuration configuration) { + Objects.requireNonNull(configuration, "configuration must not be null").values + .forEach(this::mergeValue); + return this; + } + + /** + * Sets a validated runtime value for an option. + * + * @param option target option metadata + * @param value non-null converted value + * @param originName non-blank runtime origin retained in diagnostics + * @param option value type + * @return this builder + * @throws ConfigException when the value violates the option constraint + */ + public Builder set(ConfigOption option, T value, String originName) { + Objects.requireNonNull(option, "option must not be null").validate(value); + mergeValue( + option.key(), + new Value(value, ConfigOrigin.of(ConfigSource.RUNTIME, originName))); + return this; + } + + /** + * Removes an exact key from the builder. + * + * @param key exact key to remove + * @return this builder + */ + public Builder remove(String key) { + values.remove(Objects.requireNonNull(key, "key must not be null")); + return this; + } + + /** + * Produces the immutable configuration snapshot. + * + * @return immutable snapshot containing each effective value and its origin + */ + public Configuration build() { + return values.isEmpty() ? EMPTY : new Configuration(values); + } + + private void putRaw(String key, Object value, ConfigOrigin origin) { + String normalized = Objects.requireNonNull(key, "configuration key must not be null").trim(); + if (normalized.isEmpty()) { + throw new ConfigException("Configuration key from " + origin + " must not be blank"); + } + if (value == null) { + throw new ConfigException("Configuration value for '" + normalized + "' from " + origin + " is null"); + } + mergeValue(normalized, new Value(value, origin)); + } + + private void mergeValue(String key, Value candidate) { + Value current = values.get(key); + // ConfigSource declaration order defines precedence. Equality intentionally permits + // the last layer within one source category to replace an earlier layer. + if (current == null + || candidate.origin.source().ordinal() >= current.origin.source().ordinal()) { + values.put(key, candidate); + } + } + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationFormat.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationFormat.java new file mode 100644 index 0000000000..e8f849144e --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationFormat.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.nio.file.Path; +import java.util.Locale; + +/** + * Configuration document formats supported by {@link ConfigurationParser}. + * + *

The format controls syntax parsing only. Every parser produces the same flattened, + * origin-aware {@link Configuration} representation. + */ +public enum ConfigurationFormat { + + /** YAML configuration, including nested mappings and scalar lists. */ + YAML, + + /** HOCON configuration resolved through Typesafe Config. */ + HOCON, + + /** Java properties configuration with exact, flat keys. */ + PROPERTIES; + + /** + * Infers a supported configuration format from a path extension. + * + * @param path path whose filename identifies the format + * @return inferred document format + * @throws ConfigException when the filename has no supported extension + */ + public static ConfigurationFormat fromPath(Path path) { + String filename = path.getFileName().toString().toLowerCase(Locale.ROOT); + if (filename.endsWith(".yaml") || filename.endsWith(".yml")) { + return YAML; + } + if (filename.endsWith(".conf") || filename.endsWith(".hocon")) { + return HOCON; + } + if (filename.endsWith(".properties")) { + return PROPERTIES; + } + throw new ConfigException("Unsupported configuration file format: " + path); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationLoader.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationLoader.java new file mode 100644 index 0000000000..106dfc6161 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationLoader.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +/** + * Composes named configuration layers according to the fixed StreamPark precedence model. + * + *

Layers may be registered in any order because {@link Configuration.Builder} applies source + * precedence as each value is added. Registered maps are copied immediately so callers cannot + * mutate a pending load. + */ +public final class ConfigurationLoader { + + private final Configuration.Builder builder = Configuration.builder(); + + /** + * Registers a named, defensively copied source layer. + * + * @param source source category used for precedence + * @param name non-blank source name retained in value origins + * @param values raw configuration values + * @return this loader + */ + public ConfigurationLoader add(ConfigSource source, String name, Map values) { + builder.add(name, source, values); + return this; + } + + /** + * Captures the current JVM system properties as one system-property layer. + * + *

Properties are copied when this method is called. Subsequent changes to {@link + * System#getProperties()} do not alter the registered layer. + * + * @return this loader + */ + public ConfigurationLoader addSystemProperties() { + Properties properties = System.getProperties(); + Map values = new LinkedHashMap<>(); + properties.forEach((key, value) -> values.put(String.valueOf(key), value)); + return add(ConfigSource.SYSTEM_PROPERTIES, "JVM system properties", values); + } + + /** + * Resolves every registered layer into one immutable configuration snapshot. + * + * @return immutable configuration snapshot + */ + public Configuration load() { + return builder.build(); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationParser.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationParser.java new file mode 100644 index 0000000000..f870995889 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationParser.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import org.apache.streampark.common.util.PathUtils; +import org.apache.streampark.common.util.YamlParser; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; + +/** + * Parses YAML, HOCON, and Java properties into origin-aware configuration snapshots. + * + *

YAML is loaded with SnakeYAML's safe constructor and duplicate-key rejection. Nested mappings + * are flattened with dot-separated keys; scalar lists are retained as structured values. HOCON is + * resolved before extraction. A document with a non-mapping root or ambiguous flattened keys is + * rejected instead of being interpreted heuristically. + */ +public final class ConfigurationParser { + + private ConfigurationParser() { + } + + /** + * Parses a file and infers its format from the filename. + * + * @param path configuration file path + * @return immutable file-sourced configuration snapshot + * @throws ConfigException when the file cannot be read or its extension or content is invalid + */ + public static Configuration parse(Path path) { + return parse(path, ConfigurationFormat.fromPath(path)); + } + + /** + * Parses a file using an explicit document format. + * + * @param path configuration file path + * @param format syntax used by the document + * @return immutable file-sourced configuration snapshot + * @throws ConfigException when the file cannot be read or parsed + */ + public static Configuration parse(Path path, ConfigurationFormat format) { + Objects.requireNonNull(path, "path must not be null"); + try (InputStream inputStream = PathUtils.openFile(path.toString())) { + return parse(inputStream, format, path.toAbsolutePath().normalize().toString()); + } catch (IOException e) { + throw new ConfigException("Cannot read configuration file " + path, e); + } + } + + /** + * Parses in-memory document content with a diagnostic origin name. + * + * @param content configuration document text + * @param format syntax used by the document + * @param originName source name retained in value origins and diagnostics + * @return immutable file-sourced configuration snapshot + * @throws ConfigException when the content cannot be parsed + */ + public static Configuration parse(String content, ConfigurationFormat format, String originName) { + Objects.requireNonNull(content, "configuration content must not be null"); + return parse(new StringReader(content), format, originName); + } + + /** + * Parses document content from a stream without closing the supplied stream. + * + *

YAML delegates byte decoding to {@link YamlParser} so byte-order markers are honored. + * HOCON and properties streams are decoded as UTF-8. + * + * @param inputStream document input stream + * @param format syntax used by the document + * @param originName source name retained in value origins and diagnostics + * @return immutable file-sourced configuration snapshot + * @throws ConfigException when the stream cannot be read or parsed + */ + public static Configuration parse( + InputStream inputStream, + ConfigurationFormat format, + String originName) { + Objects.requireNonNull(inputStream, "inputStream must not be null"); + if (format == ConfigurationFormat.YAML) { + return buildConfiguration(YamlParser.flatten(YamlParser.parse(inputStream)), originName); + } + return parse( + new InputStreamReader(inputStream, StandardCharsets.UTF_8), + format, + originName); + } + + private static Configuration parse(Reader reader, ConfigurationFormat format, String originName) { + Objects.requireNonNull(reader, "reader must not be null"); + Objects.requireNonNull(format, "format must not be null"); + try { + // Each syntax-specific parser ends at the same raw map boundary. Option conversion is + // deferred until typed access so this module does not require a global option registry. + Map values; + switch (format) { + case YAML: + values = parseYaml(reader); + break; + case HOCON: + values = parseHocon(reader); + break; + case PROPERTIES: + values = parseProperties(reader); + break; + default: + throw new ConfigException("Unsupported configuration format: " + format); + } + return buildConfiguration(values, originName); + } catch (ConfigException e) { + throw e; + } catch (RuntimeException | IOException e) { + throw new ConfigException("Cannot parse " + format + " configuration from " + originName, e); + } + } + + private static Map parseYaml(Reader reader) { + return YamlParser.flatten(YamlParser.parse(reader)); + } + + private static Map parseHocon(Reader reader) { + // Resolve substitutions before unwrapping to prevent unresolved ConfigValue instances from + // leaking into the engine-neutral configuration model. + Config config = ConfigFactory.parseReader(reader).resolve(); + Map result = new LinkedHashMap<>(); + config.entrySet().forEach(entry -> putUnique(result, entry.getKey(), entry.getValue().unwrapped())); + return result; + } + + private static Map parseProperties(Reader reader) throws IOException { + Properties properties = new Properties(); + properties.load(reader); + Map result = new LinkedHashMap<>(); + for (String key : properties.stringPropertyNames()) { + putUnique(result, key, properties.getProperty(key)); + } + return result; + } + + private static void putUnique(Map target, String key, Object value) { + if (target.putIfAbsent(key, value) != null) { + throw new ConfigException("Duplicate configuration key: " + key); + } + } + + private static Configuration buildConfiguration( + Map values, + String originName) { + // Parsed documents are file layers even when their content came from memory. originName + // distinguishes the physical file, compressed payload, or logical resource in diagnostics. + return Configuration.builder().add(originName, ConfigSource.FILE, values).build(); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/constants/Constants.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/Constants.java similarity index 67% rename from streampark-common/src/main/java/org/apache/streampark/common/constants/Constants.java rename to streampark-common/src/main/java/org/apache/streampark/common/configuration/Constants.java index 1282b155c1..f0861ac795 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/constants/Constants.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/Constants.java @@ -15,54 +15,48 @@ * limitations under the License. */ -package org.apache.streampark.common.constants; +package org.apache.streampark.common.configuration; -import java.time.Duration; - -/** - * A constant class to hold the constants variables. - */ +/** Shared literal values used while resolving StreamPark configuration and application artifacts. */ public final class Constants { private Constants() { } + /** Conventional name used when no explicit value is supplied. */ public static final String DEFAULT = "default"; + /** Canonical StreamPark product identifier. */ public static final String STREAM_PARK = "streampark"; + /** Plain HTTP URI scheme prefix. */ public static final String HTTP_SCHEMA = "http://"; + /** TLS-enabled HTTP URI scheme prefix. */ public static final String HTTPS_SCHEMA = "https://"; + /** Java archive filename suffix. */ public static final String JAR_SUFFIX = ".jar"; + /** ZIP archive filename suffix. */ public static final String ZIP_SUFFIX = ".zip"; - public static final String EMPTY_STRING = ""; - + /** Python source filename suffix. */ public static final String PYTHON_SUFFIX = ".py"; - public static final String SEMICOLON = ";"; - + /** Legacy mask used by existing external responses. */ public static final String DEFAULT_DATAMASK_STRING = "********"; + /** Flink entry class used to launch Python applications. */ public static final String PYTHON_FLINK_DRIVER_CLASS_NAME = "org.apache.flink.client.python.PythonDriver"; + /** StreamPark entry class used to launch Flink SQL applications. */ public static final String STREAMPARK_FLINKSQL_CLIENT_CLASS = "org.apache.streampark.flink.cli.SqlClient"; + /** StreamPark entry class used to launch Spark SQL applications. */ public static final String STREAMPARK_SPARKSQL_CLIENT_CLASS = "org.apache.streampark.spark.cli.SqlClient"; + /** Python executable path inside the packaged virtual environment. */ public static final String PYTHON_EXECUTABLE = "venv.zip/venv/bin/python3"; - public static final String SINGLE_SLASH = "/"; - - public static final String LOCAL_HOST = "localhost"; - - /** - * sleep 1000ms - */ - public static final long SLEEP_TIME_MILLIS = 1_000L; - - public static final Duration SERVER_CLOSE_WAIT_TIME = Duration.ofSeconds(3); } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/DataSize.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/DataSize.java new file mode 100644 index 0000000000..b7581657b2 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/DataSize.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.io.Serializable; +import java.util.Locale; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Immutable binary data size. + * + *

Unit suffixes are case-insensitive. {@code k}, {@code m}, {@code g}, and {@code t}, including + * their {@code b} and {@code ib} forms, use powers of 1024. Negative sizes and arithmetic overflow + * are rejected. + */ +public final class DataSize implements Comparable, Serializable { + + private static final long serialVersionUID = 1L; + private static final Pattern SIZE_PATTERN = Pattern.compile("^([+-]?\\d+)\\s*([a-zA-Z]*)$"); + + private final long bytes; + + private DataSize(long bytes) { + this.bytes = bytes; + } + + /** + * Creates a data size from bytes. + * + * @param bytes non-negative byte count + * @return immutable data size + * @throws IllegalArgumentException when {@code bytes} is negative + */ + public static DataSize ofBytes(long bytes) { + if (bytes < 0) { + throw new IllegalArgumentException("data size must not be negative"); + } + return new DataSize(bytes); + } + + /** + * Creates a data size from mebibytes. + * + * @param mebiBytes non-negative mebibyte count + * @return immutable data size + * @throws ArithmeticException when conversion overflows a {@code long} + */ + public static DataSize ofMebiBytes(long mebiBytes) { + return ofBytes(Math.multiplyExact(mebiBytes, 1024L * 1024L)); + } + + /** + * Parses a non-negative binary data size. + * + *

Bare numbers represent bytes. The suffixes {@code k}, {@code m}, {@code g}, and {@code t}, + * with optional {@code b} or {@code ib}, use binary multiples. + * + * @param value textual data size + * @return parsed data size + * @throws IllegalArgumentException when the value is negative, malformed, or uses an unknown + * unit + * @throws ArithmeticException when conversion overflows a {@code long} + */ + public static DataSize parse(String value) { + String text = Objects.requireNonNull(value, "data size must not be null").trim(); + Matcher matcher = SIZE_PATTERN.matcher(text); + if (!matcher.matches()) { + throw new IllegalArgumentException("invalid data size: " + value); + } + long amount = Long.parseLong(matcher.group(1)); + String unit = matcher.group(2).toLowerCase(Locale.ROOT); + long multiplier; + switch (unit) { + case "": + case "b": + multiplier = 1L; + break; + case "k": + case "kb": + case "kib": + multiplier = 1024L; + break; + case "m": + case "mb": + case "mib": + multiplier = 1024L * 1024L; + break; + case "g": + case "gb": + case "gib": + multiplier = 1024L * 1024L * 1024L; + break; + case "t": + case "tb": + case "tib": + multiplier = 1024L * 1024L * 1024L * 1024L; + break; + default: + throw new IllegalArgumentException("unsupported data size unit: " + unit); + } + return ofBytes(Math.multiplyExact(amount, multiplier)); + } + + /** + * Returns the exact number of bytes. + * + * @return byte count + */ + public long bytes() { + return bytes; + } + + @Override + public int compareTo(DataSize other) { + return Long.compare(bytes, other.bytes); + } + + @Override + public boolean equals(Object object) { + return object instanceof DataSize && bytes == ((DataSize) object).bytes; + } + + @Override + public int hashCode() { + return Long.hashCode(bytes); + } + + @Override + public String toString() { + return bytes + "b"; + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/FlinkOptions.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/FlinkOptions.java new file mode 100644 index 0000000000..fc7bc15a03 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/FlinkOptions.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import org.apache.streampark.common.configuration.option.ApplicationOptions; + +/** + * Flink-specific application options and configuration namespaces. + * + *

Options in this catalog describe StreamPark's application contract. Native Flink properties + * remain ordinary Flink configuration and are selected by namespace only when launcher or table + * configuration views are assembled. Reusing engine-neutral options keeps shared keys identical + * across Flink, Spark, console, and submission modules. + */ +public final class FlinkOptions { + + /** Engine-neutral application configuration resource. */ + public static final ConfigOption APPLICATION_CONFIG = ApplicationOptions.CONFIG; + + /** Engine-neutral application name. */ + public static final ConfigOption APPLICATION_NAME = ApplicationOptions.NAME; + + /** Engine-neutral SQL text or resource. */ + public static final ConfigOption SQL = ApplicationOptions.SQL; + + /** Compressed native Flink YAML transported to a Table API application. */ + public static final ConfigOption FLINK_CONFIGURATION = + stringOption("flink.conf", "Compressed Flink configuration for table applications."); + + /** Parsing mode retained when compressed Flink YAML no longer has a source filename. */ + public static final ConfigOption FLINK_CONFIGURATION_STANDARD_YAML = + ConfigOptions.key("flink.conf.standard-yaml") + .booleanType() + .defaultValue(true) + .withDescription( + "Whether the compressed Flink configuration uses standard rather than legacy YAML.") + .build(); + + /** Default parallelism applied to a Flink job. */ + public static final ConfigOption PARALLELISM = + ConfigOptions.key("parallelism.default") + .intType() + .noDefaultValue() + .check(value -> value > 0, "parallelism must be greater than zero") + .withDescription("Default Flink job parallelism.") + .build(); + + /** Pipeline name forwarded to native Flink configuration. */ + public static final ConfigOption PIPELINE_NAME = + stringOption("pipeline.name", "Flink pipeline name."); + + /** Planner selected while constructing the Table API environment. */ + public static final ConfigOption TABLE_PLANNER = + stringOption("flink.table.planner", "Planner selected for the Table API environment."); + + /** Runtime mode selected while constructing the Table API environment. */ + public static final ConfigOption TABLE_MODE = + stringOption("flink.table.mode", "Runtime mode selected for the Table API environment."); + + /** Initial catalog selected for the Table API environment. */ + public static final ConfigOption TABLE_CATALOG = + stringOption("flink.table.catalog", "Built-in Table API catalog name."); + + /** Initial database selected for the Table API environment. */ + public static final ConfigOption TABLE_DATABASE = + stringOption("flink.table.database", "Built-in Table API database name."); + + /** Container image pull policy forwarded to Kubernetes deployments. */ + public static final ConfigOption KUBERNETES_IMAGE_PULL_POLICY = + stringOption( + "kubernetes.container.image.pull-policy", + "Kubernetes image pull policy forwarded to Flink deployment."); + + /** Engine-neutral application entry class. */ + public static final ConfigOption APPLICATION_MAIN_CLASS = ApplicationOptions.MAIN_CLASS; + + /** Namespace for Flink launcher options consumed by StreamPark scripts. */ + public static final ConfigPrefix OPTION_PREFIX = ConfigPrefix.of("flink.option."); + + /** Namespace for native Flink dynamic properties. */ + public static final ConfigPrefix PROPERTY_PREFIX = ConfigPrefix.of("flink.property."); + + /** Namespace for StreamPark Table API initialization options. */ + public static final ConfigPrefix TABLE_PREFIX = ConfigPrefix.of("flink.table."); + + private FlinkOptions() { + } + + private static ConfigOption stringOption(String key, String description) { + return ConfigOptions.key(key) + .stringType() + .noDefaultValue() + .withDescription(description) + .build(); + } +} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/FlinkRunOption.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/FlinkRunOption.java similarity index 71% rename from streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/FlinkRunOption.java rename to streampark-common/src/main/java/org/apache/streampark/common/configuration/FlinkRunOption.java index 4d0b66d0fc..060d48ff58 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/FlinkRunOption.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/FlinkRunOption.java @@ -15,52 +15,114 @@ * limitations under the License. */ -package org.apache.streampark.flink.core.conf; +package org.apache.streampark.common.configuration; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; -/** Applies to all optional parameters under flink run. */ +/** + * Defines the Commons CLI option metadata used by StreamPark's Flink submission flow. + * + *

The class provides reusable option sets and parsing helpers for Flink launcher arguments. It + * describes command-line syntax only; runtime configuration keys and values remain the + * responsibility of {@link FlinkOptions} and the calling submission component. + */ public final class FlinkRunOption { private static final String SAVEPOINT_PATH_ARG = "savepointPath"; + /** Prints launcher help. */ public static final Option HELP_OPTION = new Option("h", "help", false, null); + + /** Selects the application JAR. */ public static final Option JAR_OPTION = new Option("j", "jarfile", true, null); + + /** Selects the application entry class. */ public static final Option CLASS_OPTION = new Option("c", "class", true, null); + + /** Adds a dependency URL to the application classpath. */ public static final Option CLASSPATH_OPTION = new Option("C", "classpath", true, null); + + /** Sets the application parallelism. */ public static final Option PARALLELISM_OPTION = new Option("p", "parallelism", true, null); + + /** Submits the application in detached mode. */ public static final Option DETACHED_OPTION = new Option("d", "detached", false, null); + + /** Shuts down the cluster when an attached client exits. */ public static final Option SHUTDOWN_IF_ATTACHED_OPTION = new Option("sae", "shutdownOnAttachedExit", false, null); + + /** Uses the historical YARN detached-mode flag. */ public static final Option YARN_DETACHED_OPTION = new Option("yd", "yarndetached", false, null); + + /** Supplies arguments to the application main method. */ public static final Option ARGS_OPTION = new Option("a", "arguments", true, null); + + /** Selects a JobManager endpoint. */ public static final Option ADDRESS_OPTION = new Option("m", "jobmanager", true, null); + + /** Selects the savepoint used to restore an application. */ public static final Option SAVEPOINT_PATH_OPTION = new Option("s", "fromSavepoint", true, null); + + /** Permits restore when savepoint state cannot be mapped to every operator. */ public static final Option SAVEPOINT_ALLOW_NON_RESTORED_OPTION = new Option("n", "allowNonRestoredState", false, null); + + /** Selects a savepoint to dispose. */ public static final Option SAVEPOINT_DISPOSE_OPTION = new Option("d", "dispose", true, null); + + /** Restricts a query to running jobs. */ public static final Option RUNNING_OPTION = new Option("r", "running", false, null); + + /** Restricts a query to scheduled jobs. */ public static final Option SCHEDULED_OPTION = new Option("s", "scheduled", false, null); + + /** Selects all matching jobs. */ public static final Option ALL_OPTION = new Option("a", "all", false, null); + + /** Sets the high-availability ZooKeeper namespace. */ public static final Option ZOOKEEPER_NAMESPACE_OPTION = new Option("z", "zookeeperNamespace", true, null); + + /** Creates a savepoint while cancelling an application. */ public static final Option CANCEL_WITH_SAVEPOINT_OPTION = new Option("s", "withSavepoint", true, null); + + /** Selects the savepoint directory used while stopping an application. */ public static final Option STOP_WITH_SAVEPOINT_PATH = new Option("p", SAVEPOINT_PATH_ARG, true, null); + + /** Drains event-time input before stopping an application. */ public static final Option STOP_AND_DRAIN = new Option("d", "drain", false, null); + + /** Selects a Python application file. */ public static final Option PY_OPTION = new Option("py", "python", true, null); + + /** Adds Python source files or archives. */ public static final Option PYFILES_OPTION = new Option("pyfs", "pyFiles", true, null); + + /** Selects a Python module as the application entry point. */ public static final Option PYMODULE_OPTION = new Option("pym", "pyModule", true, null); + + /** Supplies a Python requirements specification. */ public static final Option PYREQUIREMENTS_OPTION = new Option("pyreq", "pyRequirements", true, null); + + /** Adds archives to the Python worker environment. */ public static final Option PYARCHIVE_OPTION = new Option("pyarch", "pyArchives", true, null); + + /** Selects the Python worker executable. */ public static final Option PYEXEC_OPTION = new Option("pyexec", "pyExecutable", true, null); + + /** Selects a Flink executor implementation. */ public static final Option EXECUTOR_OPTION = new Option("e", "executor", true, null); + + /** Selects the Flink deployment target. */ public static final Option TARGET_OPTION = new Option("t", "target", true, null); + /** Supplies repeatable native Flink dynamic properties. */ public static final Option DYNAMIC_PROPERTIES = Option.builder("D") .argName("property=value") @@ -131,6 +193,11 @@ public final class FlinkRunOption { private FlinkRunOption() { } + /** + * Returns the union of general run and YARN launcher options. + * + * @return new Commons CLI schema containing all supported launcher options + */ public static Options allOptions() { Options commOptions = getRunCommandOptions(); Options yarnOptions = getYARNOptions(); @@ -146,6 +213,11 @@ public static Options allOptions() { return resultOptions; } + /** + * Returns options accepted by the general Flink run command. + * + * @return new Commons CLI schema for a Flink run command + */ public static Options getRunCommandOptions() { Options options = buildGeneralOptions(new Options()); options = getProgramSpecificOptions(options); @@ -157,6 +229,11 @@ public static Options getRunCommandOptions() { return options; } + /** + * Returns YARN-specific launcher options. + * + * @return new Commons CLI schema for YARN launcher flags + */ public static Options getYARNOptions() { Options allOptions = new Options(); allOptions.addOption(DETACHED_OPTION); @@ -164,12 +241,24 @@ public static Options getYARNOptions() { return allOptions; } + /** + * Adds general launcher options to a supplied schema. + * + * @param options target Commons CLI schema + * @return the same schema after mutation + */ public static Options buildGeneralOptions(Options options) { options.addOption(HELP_OPTION); options.addOption(new Option("v", "verbose", false, "This option is deprecated.")); return options; } + /** + * Adds program-specific launcher options to a supplied schema. + * + * @param options target Commons CLI schema + * @return the same schema after mutation + */ public static Options getProgramSpecificOptions(Options options) { options.addOption(JAR_OPTION); options.addOption(CLASS_OPTION); @@ -189,6 +278,13 @@ public static Options getProgramSpecificOptions(Options options) { return options; } + /** + * Returns a new schema containing options from both inputs. + * + * @param optionsA first source schema + * @param optionsB second source schema + * @return new combined schema + */ public static Options mergeOptions(Options optionsA, Options optionsB) { Options resultOptions = new Options(); if (optionsA == null || optionsB == null) { @@ -203,12 +299,21 @@ public static Options mergeOptions(Options optionsA, Options optionsB) { return resultOptions; } + /** + * Parses launcher arguments and reports syntax errors as configuration failures. + * + * @param options Commons CLI schema used for validation + * @param args tokenized launcher arguments + * @param stopAtNonOptions whether parsing stops at the first non-option token + * @return parsed command line + * @throws ConfigException when Commons CLI rejects the arguments + */ public static CommandLine parse(Options options, String[] args, boolean stopAtNonOptions) { DefaultParser parser = new DefaultParser(); try { return parser.parse(options, args, stopAtNonOptions); } catch (Exception e) { - throw new RuntimeException(e); + throw new ConfigException("Invalid Flink launcher arguments", e); } } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/GlobalConfiguration.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/GlobalConfiguration.java new file mode 100644 index 0000000000..6ffac8a4f1 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/GlobalConfiguration.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Atomic process-wide reference to the current immutable server configuration snapshot. + * + *

The global reference exists only at the process boundary. Domain code should capture + * {@link #current()} once at the beginning of a multi-step operation and pass that snapshot to + * derived objects. Replacing or overlaying configuration never mutates an existing snapshot. + */ +public final class GlobalConfiguration { + + private static final AtomicReference CURRENT = + new AtomicReference<>(new ConfigurationLoader().addSystemProperties().load()); + + private GlobalConfiguration() { + } + + /** + * Returns the current immutable process snapshot. + * + * @return current snapshot + */ + public static Configuration current() { + return CURRENT.get(); + } + + /** + * Atomically update the process snapshot. + * + * @param configuration new immutable snapshot + */ + public static void update(Configuration configuration) { + CURRENT.set(Objects.requireNonNull(configuration, "configuration must not be null")); + } + + /** + * Atomically overlays a named source layer while honoring source precedence. + * + * @param source source category used for precedence + * @param name non-blank origin name retained in diagnostics + * @param values raw values to overlay + */ + public static void overlay(ConfigSource source, String name, Map values) { + Objects.requireNonNull(source, "source must not be null"); + CURRENT.updateAndGet( + current -> Configuration.builder(current).add(name, source, values).build()); + } + + /** + * Atomically sets a validated runtime option. + * + * @param option target option metadata + * @param value non-null converted value + * @param originName non-blank runtime origin retained in diagnostics + * @param option value type + */ + public static void set(ConfigOption option, T value, String originName) { + CURRENT.updateAndGet( + current -> Configuration.builder(current).set(option, value, originName).build()); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/GroupedOptionsParser.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/GroupedOptionsParser.java new file mode 100644 index 0000000000..88a7012153 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/GroupedOptionsParser.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; + +/** + * Parses repeated {@code --group key=value} arguments used by quick-start connectors. + * + *

Each group retains insertion order and may contain multiple distinct keys. The returned outer + * and inner maps are immutable so connector initialization observes a stable argument snapshot. + */ +public final class GroupedOptionsParser { + + private GroupedOptionsParser() { + } + + /** + * Parses group and entry pairs from a tokenized argument array. + * + * @param arguments alternating {@code --group} and {@code key=value} tokens + * @return immutable group-to-entry mapping + * @throws ConfigException when a pair is incomplete, malformed, or duplicates a key in a group + */ + public static Map> parse(String[] arguments) { + Objects.requireNonNull(arguments, "arguments must not be null"); + Map> groups = new LinkedHashMap<>(); + for (int index = 0; index < arguments.length; index += 2) { + String group = arguments[index]; + if (!group.startsWith(CommandLineParser.LONG_OPTION_PREFIX) + || group.length() == CommandLineParser.LONG_OPTION_PREFIX.length()) { + throw new ConfigException("Expected a --group argument but found '" + group + "'"); + } + if (index + 1 >= arguments.length) { + throw new ConfigException("Missing key=value for group " + group); + } + String expression = arguments[index + 1]; + int separator = expression.indexOf('='); + if (separator <= 0) { + throw new ConfigException("Grouped option must use key=value syntax: " + expression); + } + String groupName = group.substring(CommandLineParser.LONG_OPTION_PREFIX.length()); + String key = expression.substring(0, separator).trim(); + String value = expression.substring(separator + 1); + Map entries = + groups.computeIfAbsent(groupName, ignored -> new LinkedHashMap<>()); + if (entries.putIfAbsent(key, value) != null) { + throw new ConfigException("Duplicate grouped option: " + groupName + "." + key); + } + } + + // Freeze both map levels; wrapping only the outer map would still allow connectors to + // mutate entries belonging to another initialization stage. + Map> result = new LinkedHashMap<>(); + groups.forEach( + (group, values) -> result.put(group, Collections.unmodifiableMap(new LinkedHashMap<>(values)))); + return Collections.unmodifiableMap(result); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/JvmOptionsParser.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/JvmOptionsParser.java new file mode 100644 index 0000000000..45dc47081c --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/JvmOptionsParser.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Parses JVM {@code -Dkey=value} options into a configuration snapshot. + * + *

Both {@code -Dkey=value} and the token pair {@code -D key=value} are accepted. The parser does + * not modify {@link System#getProperties()}; it returns an immutable command-line layer that can be + * composed explicitly with other sources. + */ +public final class JvmOptionsParser { + + private static final String PREFIX = "-D"; + + private JvmOptionsParser() { + } + + /** + * Parses a JVM option string without invoking a shell. + * + * @param options JVM property options, optionally containing quoted values + * @return immutable command-line configuration snapshot + * @throws ConfigException when a token is not a {@code -D} property or duplicates a key + */ + public static Configuration parse(String options) { + List tokens = CommandLineTokenizer.tokenize(options); + Map properties = new LinkedHashMap<>(); + for (int index = 0; index < tokens.size(); index++) { + String token = tokens.get(index); + String expression; + if (PREFIX.equals(token)) { + if (++index >= tokens.size()) { + throw new ConfigException("-D must be followed by key=value"); + } + expression = tokens.get(index); + } else if (token.startsWith(PREFIX)) { + expression = token.substring(PREFIX.length()); + } else { + throw new ConfigException("Expected a -Dkey=value option but found '" + token + "'"); + } + + int separator = expression.indexOf('='); + if (separator <= 0) { + throw new ConfigException("JVM property must use -Dkey=value syntax: " + token); + } + String key = expression.substring(0, separator).trim(); + String value = expression.substring(separator + 1); + if (properties.putIfAbsent(key, value) != null) { + throw new ConfigException("Duplicate JVM property: " + key); + } + } + return Configuration.builder() + .add("JVM options", ConfigSource.COMMAND_LINE, properties) + .build(); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ReadableConfig.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ReadableConfig.java new file mode 100644 index 0000000000..455859f14e --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ReadableConfig.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.util.Optional; + +/** + * Read-only typed access to a configuration snapshot. + * + *

Consumers depend on this interface when they only require option resolution and should not + * need to know how layers were parsed or composed. Implementations resolve canonical keys before + * fallback keys and apply the type conversion and validation declared by each option. + */ +public interface ReadableConfig { + + /** + * Returns the effective value of an option. + * + * @param option option metadata used for lookup, conversion, and validation + * @param option value type + * @return configured value, or the declared default when the option is absent + * @throws ConfigException when a required option is absent or its value is invalid + */ + T get(ConfigOption option); + + /** + * Returns the explicitly configured value of an option. + * + *

Defaults are intentionally not returned, which allows callers to distinguish an omitted + * option from one whose effective value happens to equal its default. + * + * @param option option metadata used for lookup, conversion, and validation + * @param option value type + * @return configured value, or an empty optional when neither canonical nor fallback key exists + * @throws ConfigException when the configured value cannot be converted or validated + */ + Optional getOptional(ConfigOption option); + + /** + * Returns whether an option is explicitly present under its canonical or a fallback key. + * + * @param option option metadata used for lookup + * @return {@code true} when a configured value exists; {@code false} for defaults alone + */ + boolean contains(ConfigOption option); +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/SensitiveKeys.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/SensitiveKeys.java new file mode 100644 index 0000000000..c2f4084d38 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/SensitiveKeys.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import java.util.Locale; +import java.util.Objects; +import java.util.Set; + +/** + * Central policy for configuration values that must never be rendered in clear text. + * + *

The policy supplements the explicit {@link ConfigOptions.TypedOptionBuilder#sensitive()} + * marker for untyped or engine-native configuration. Matching is intentionally conservative: + * false positives mask diagnostics, while false negatives can expose credentials in logs. + */ +public final class SensitiveKeys { + + /** Stable replacement used by diagnostics and redacted map views. */ + public static final String MASK = "******"; + + /** + * Markers cover Flink's native redaction policy and StreamPark-specific credential names. + * Substring matching intentionally favors masking over exposing an unknown secret convention. + */ + private static final Set MARKERS = Set.of( + "password", + "passwd", + "secret", + "fs.azure.account.key", + "apikey", + "api-key", + "api.key", + "auth-params", + "service-key", + "token", + "basic-auth", + "jaas.config", + "http-headers", + "access-key", + "access.key", + "accesskey", + "credential", + "keytab", + "private-key", + "private.key"); + + private SensitiveKeys() { + } + + /** + * Returns whether a configuration key conventionally identifies secret material. + * + * @param key configuration key to inspect + * @return {@code true} when the lower-cased key contains a supported sensitive marker + */ + public static boolean isSensitive(String key) { + String normalized = Objects.requireNonNull(key, "key must not be null") + .toLowerCase(Locale.ROOT); + return MARKERS.stream().anyMatch(normalized::contains); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/Workspace.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/Workspace.java new file mode 100644 index 0000000000..2ad5b6040c --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/Workspace.java @@ -0,0 +1,262 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import org.apache.streampark.common.configuration.option.WorkspaceOptions; +import org.apache.streampark.common.enums.StorageType; +import org.apache.streampark.common.util.HdfsUtils; + +import java.net.URI; +import java.util.Objects; + +/** + * Immutable path layout for StreamPark's local and remote workspaces. + * + *

{@link #LOCAL} and {@link #REMOTE} provide constant-style access for normal application code. + * Both instances are derived from the same global configuration snapshot when this class is first + * initialized. Configuration bootstrap code must therefore call {@link + * #verifyInitializedFrom(Configuration)} after all configuration sources have been composed. + * Operations that intentionally use another immutable snapshot can create an independent layout + * through {@link #of(StorageType, Configuration)}. + */ +public final class Workspace { + + private static final Configuration INITIAL_CONFIGURATION = GlobalConfiguration.current(); + + /** Local workspace layout derived from the process configuration. */ + public static final Workspace LOCAL = new Workspace(StorageType.LFS, INITIAL_CONFIGURATION); + + /** Remote workspace layout derived from the process configuration. */ + public static final Workspace REMOTE = new Workspace(StorageType.HDFS, INITIAL_CONFIGURATION); + + /** Local directory containing assembled application distributions. */ + public static final String APP_LOCAL_DIST = child(LOCAL.root, "dist"); + + /** Local Maven repository used by the built-in compilation pipeline. */ + public static final String MAVEN_LOCAL_PATH = child(LOCAL.root, "mvnrepo"); + + /** Local directory containing checked-out project sources. */ + public static final String PROJECT_LOCAL_PATH = child(LOCAL.root, "project"); + + /** Local directory containing StreamPark logs. */ + public static final String LOG_LOCAL_PATH = child(LOCAL.root, "logs"); + + /** Local directory containing project build logs. */ + public static final String PROJECT_BUILD_LOG_PATH = child(LOG_LOCAL_PATH, "build_logs"); + + /** Remote directory consumed by the Flink history server. */ + public static final String ARCHIVES_FILE_PATH = child(REMOTE.root, "historyserver/archive"); + + /** Configured root from which every path in this layout is derived. */ + public final String root; + + /** Directory containing StreamPark plugins. */ + public final String plugins; + + /** Directory containing framework client artifacts. */ + public final String client; + + /** Directory containing version-specific Flink shim artifacts. */ + public final String shims; + + /** Directory containing user-uploaded artifacts. */ + public final String uploads; + + /** Directory containing Python runtime artifacts. */ + public final String python; + + /** Python virtual-environment archive distributed to PyFlink jobs. */ + public final String pythonVenv; + + /** Directory containing isolated per-application workspaces. */ + public final String workspace; + + /** Directory containing managed Flink distributions. */ + public final String flink; + + /** Directory containing managed Spark distributions. */ + public final String spark; + + /** Directory containing application backups. */ + public final String backups; + + /** Default directory containing Flink savepoints. */ + public final String savepoints; + + /** Directory containing StreamPark application JARs. */ + public final String jars; + + private Workspace(StorageType storageType, Configuration configuration) { + Objects.requireNonNull(storageType, "storageType must not be null"); + Objects.requireNonNull(configuration, "configuration must not be null"); + root = resolveRoot(storageType, configuration); + plugins = child(root, "plugins"); + client = child(root, "client"); + shims = child(root, "shims"); + uploads = child(root, "uploads"); + python = child(root, "python"); + pythonVenv = child(python, "venv.zip"); + workspace = child(root, "workspace"); + flink = child(root, "flink"); + spark = child(root, "spark"); + backups = child(root, "backups"); + savepoints = child(root, "savepoints"); + jars = child(root, "jars"); + } + + /** + * Creates an independent workspace layout from a stable configuration snapshot. + * + * @param storageType local or remote storage type + * @param configuration stable configuration snapshot + * @return immutable workspace layout + */ + public static Workspace of(StorageType storageType, Configuration configuration) { + return new Workspace(storageType, configuration); + } + + /** + * Verifies that the constant layouts were initialized from the completed bootstrap snapshot. + * + *

The check converts accidental early class initialization into an actionable startup + * failure instead of allowing the process to operate on stale workspace paths. + * + * @param configuration completed bootstrap configuration + * @throws ConfigException if either constant was initialized from another workspace root + */ + public static void verifyInitializedFrom(Configuration configuration) { + Workspace expectedLocal = of(StorageType.LFS, configuration); + Workspace expectedRemote = of(StorageType.HDFS, configuration); + if (!LOCAL.root.equals(expectedLocal.root) || !REMOTE.root.equals(expectedRemote.root)) { + throw new ConfigException( + "Workspace constants were initialized before configuration bootstrap completed"); + } + } + + private static String resolveRoot(StorageType storageType, Configuration configuration) { + if (storageType == StorageType.LFS) { + return normalizeRoot(configuration.get(WorkspaceOptions.LOCAL_ROOT)); + } + + String configured = normalizeRoot(configuration.get(WorkspaceOptions.REMOTE_ROOT)); + URI configuredUri = parseUri(configured, WorkspaceOptions.REMOTE_ROOT.key()); + rejectUnsupportedUriComponents(configuredUri, WorkspaceOptions.REMOTE_ROOT.key()); + if (configuredUri.getScheme() == null) { + return qualifyWithDefaultFileSystem(configured); + } + if (!"hdfs".equalsIgnoreCase(configuredUri.getScheme())) { + throw new ConfigException( + "Option '" + + WorkspaceOptions.REMOTE_ROOT.key() + + "' must use the hdfs scheme: " + + configured); + } + return validateAndQualifyHdfsUri(configured, configuredUri); + } + + private static String validateAndQualifyHdfsUri(String configured, URI configuredUri) { + String defaultFileSystem = defaultFileSystem(); + URI defaultUri = parseUri(defaultFileSystem, "fs.defaultFS"); + if (!"hdfs".equalsIgnoreCase(defaultUri.getScheme())) { + throw new ConfigException( + "Remote workspace uses HDFS but fs.defaultFS is '" + defaultFileSystem + "'"); + } + + String configuredAuthority = configuredUri.getAuthority(); + if (configuredAuthority == null) { + return qualify(defaultFileSystem, configuredUri.getPath()); + } + String defaultAuthority = defaultUri.getAuthority(); + if (defaultAuthority == null || !configuredAuthority.equalsIgnoreCase(defaultAuthority)) { + throw new ConfigException( + "Remote workspace authority '" + + configuredAuthority + + "' does not match fs.defaultFS '" + + defaultFileSystem + + "'"); + } + return configured; + } + + private static String qualifyWithDefaultFileSystem(String configured) { + return qualify(defaultFileSystem(), configured); + } + + private static String qualify(String defaultFileSystem, String path) { + String relativePath = path; + while (relativePath.startsWith("/")) { + relativePath = relativePath.substring(1); + } + return relativePath.isEmpty() ? defaultFileSystem : child(defaultFileSystem, relativePath); + } + + private static String defaultFileSystem() { + String value = HdfsUtils.getDefaultFS(); + if (value == null || value.trim().isEmpty()) { + throw new ConfigException("Hadoop property 'fs.defaultFS' must not be blank"); + } + String normalized = normalizeRoot(value); + URI uri = parseUri(normalized, "fs.defaultFS"); + if (uri.getScheme() == null) { + throw new ConfigException("Hadoop property 'fs.defaultFS' must contain a URI scheme"); + } + return normalized; + } + + private static void rejectUnsupportedUriComponents(URI uri, String key) { + if (uri.getQuery() != null || uri.getFragment() != null) { + throw new ConfigException("Option '" + key + "' must not contain a query or fragment"); + } + } + + private static URI parseUri(String value, String key) { + try { + return URI.create(value); + } catch (IllegalArgumentException exception) { + throw new ConfigException("Option '" + key + "' is not a valid URI: " + value, exception); + } + } + + private static String child(String parent, String name) { + return parent.endsWith("/") ? parent + name : parent + "/" + name; + } + + private static String normalizeRoot(String value) { + String normalized = value.trim(); + if (isSchemeRoot(normalized)) { + return normalized; + } + while (normalized.length() > 1 && normalized.endsWith("/")) { + normalized = normalized.substring(0, normalized.length() - 1); + } + return normalized; + } + + private static boolean isSchemeRoot(String value) { + int separator = value.indexOf(':'); + if (separator <= 0 || separator == value.length() - 1) { + return false; + } + for (int index = separator + 1; index < value.length(); index++) { + if (value.charAt(index) != '/') { + return false; + } + } + return true; + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/ApplicationOptions.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/ApplicationOptions.java new file mode 100644 index 0000000000..3ccfc5f151 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/ApplicationOptions.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration.option; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; +import org.apache.streampark.common.configuration.ConfigPrefix; + +/** + * Engine-neutral application arguments and configuration namespaces. + * + *

These options form the shared submission contract used before an application is adapted to a + * Flink- or Spark-specific runtime. + */ +public final class ApplicationOptions { + + /** Application configuration resource supplied to an engine runtime. */ + public static final ConfigOption CONFIG = + ConfigOptions.key("conf") + .stringType() + .noDefaultValue() + .withDescription("Application configuration resource.") + .build(); + + /** Logical application name shared by submission and runtime components. */ + public static final ConfigOption NAME = + ConfigOptions.key("app.name") + .stringType() + .noDefaultValue() + .withDescription("Application name.") + .build(); + + /** SQL text or resource supplied to a SQL application. */ + public static final ConfigOption SQL = + ConfigOptions.key("sql") + .stringType() + .noDefaultValue() + .withDescription("SQL text or SQL resource supplied to an application.") + .build(); + + /** Internal application entry class resolved by the submission client. */ + public static final ConfigOption MAIN_CLASS = + ConfigOptions.key("$internal.application.main") + .stringType() + .noDefaultValue() + .withDescription("Resolved application entry class used by a submission client.") + .build(); + + /** Namespace containing general application properties. */ + public static final ConfigPrefix APPLICATION_PREFIX = ConfigPrefix.of("app."); + + /** Namespace containing SQL application properties. */ + public static final ConfigPrefix SQL_PREFIX = ConfigPrefix.of("sql."); + + private ApplicationOptions() { + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/WeChatAlertForm.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/CoreOptions.java similarity index 51% rename from streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/WeChatAlertForm.java rename to streampark-common/src/main/java/org/apache/streampark/common/configuration/option/CoreOptions.java index 2cb16e9bb2..0339886125 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/WeChatAlertForm.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/CoreOptions.java @@ -15,30 +15,26 @@ * limitations under the License. */ -package org.apache.streampark.e2e.pages.setting.alarm; +package org.apache.streampark.common.configuration.option; -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; - -@Getter -public class WeChatAlertForm extends CommonForm { - - public WebDriver driver; - - @FindBy(id = "form_item_weToken") - public WebElement inputWeChatToken; +/** + * Process bootstrap options required before other StreamPark services are initialized. + * + *

Declarations in this class must remain free of environment reads and filesystem access. + */ +public final class CoreOptions { - public WeChatAlertForm(AlertTypeDetailForm alertTypeDetailForm) { - super(alertTypeDetailForm); - this.driver = alertTypeDetailForm.driver; - } + /** StreamPark installation directory used to resolve distribution resources. */ + public static final ConfigOption APP_HOME = + ConfigOptions.key("app.home") + .stringType() + .noDefaultValue() + .withDescription("StreamPark installation directory.") + .build(); - public WeChatAlertForm token(String token) { - WebElementDeleteAndInput(inputWeChatToken, token); - return this; + private CoreOptions() { } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/DeploymentOptions.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/DeploymentOptions.java new file mode 100644 index 0000000000..c37ee7e359 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/DeploymentOptions.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration.option; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; + +/** + * Engine-neutral deployment metadata passed between console and submission clients. + * + *

Options represent transport contracts only; deployment-mode validation belongs to the + * submission client that understands the selected engine and cluster manager. + */ +public final class DeploymentOptions { + + /** YARN application identifier returned by the ResourceManager. */ + public static final ConfigOption YARN_APPLICATION_ID = string("yarn.application.id"); + + /** User-facing YARN application name. */ + public static final ConfigOption YARN_APPLICATION_NAME = string("yarn.application.name"); + + /** YARN scheduler queue selected for submission. */ + public static final ConfigOption YARN_QUEUE = string("yarn.application.queue"); + + /** YARN node label expression selected for submission. */ + public static final ConfigOption YARN_NODE_LABEL = string("yarn.application.node-label"); + + /** Kubernetes service account assigned to submitted workloads. */ + public static final ConfigOption KUBERNETES_SERVICE_ACCOUNT = + string("kubernetes.service-account"); + + private DeploymentOptions() { + } + + private static ConfigOption string(String key) { + return ConfigOptions.key(key) + .stringType() + .noDefaultValue() + .withDescription("Deployment parameter '" + key + "'.") + .build(); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/DockerOptions.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/DockerOptions.java new file mode 100644 index 0000000000..70a2f26a2d --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/DockerOptions.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration.option; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; + +/** + * Docker HTTP client options used by the application packer. + * + *

Timeout options retain {@link Duration} values after conversion, preventing individual Docker + * clients from interpreting bare numeric units independently. + */ +public final class DockerOptions { + + /** Docker daemon endpoint; an empty value delegates discovery to the client. */ + public static final ConfigOption HOST = + ConfigOptions.key("streampark.docker.http-client.docker-host") + .stringType() + .defaultValue("") + .withDescription("Docker daemon endpoint; an empty value uses the client default.") + .build(); + + /** Maximum number of concurrent Docker HTTP connections. */ + public static final ConfigOption MAX_CONNECTIONS = + ConfigOptions.key("streampark.docker.http-client.max-connections") + .intType() + .defaultValue(100) + .check(value -> value > 0, "maximum connections must be greater than zero") + .withDescription("Maximum number of Docker HTTP client connections.") + .build(); + + /** Timeout for establishing a Docker daemon connection. */ + public static final ConfigOption CONNECTION_TIMEOUT = + ConfigOptions.key("streampark.docker.http-client.connection-timeout-sec") + .durationType(ChronoUnit.SECONDS) + .defaultValue(Duration.ofSeconds(100)) + .check(value -> !value.isZero(), "connection timeout must be greater than zero") + .withDescription("Docker HTTP client connection timeout.") + .build(); + + /** Timeout for receiving a Docker daemon response. */ + public static final ConfigOption RESPONSE_TIMEOUT = + ConfigOptions.key("streampark.docker.http-client.response-timeout-sec") + .durationType(ChronoUnit.SECONDS) + .defaultValue(Duration.ofSeconds(120)) + .check(value -> !value.isZero(), "response timeout must be greater than zero") + .withDescription("Docker HTTP client response timeout.") + .build(); + + private DockerOptions() { + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/HadoopOptions.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/HadoopOptions.java new file mode 100644 index 0000000000..5145aff731 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/HadoopOptions.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration.option; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; + +import java.time.Duration; + +/** + * Hadoop filesystem identity and Kerberos authentication options. + * + *

Credential-bearing options are marked sensitive at declaration time so conversion failures + * never include their raw values. + */ +public final class HadoopOptions { + + /** JVM property consumed by Hadoop clients when no explicit UGI is supplied. */ + public static final String HADOOP_USER_NAME_PROPERTY = "HADOOP_USER_NAME"; + + /** Hadoop user used by StreamPark services when Kerberos is disabled. */ + public static final ConfigOption USER_NAME = + ConfigOptions.key("streampark.hadoop-user-name") + .stringType() + .defaultValue("hdfs") + .check(value -> !value.trim().isEmpty(), "Hadoop user name must not be blank") + .withDescription("Hadoop user used by StreamPark services.") + .build(); + + /** Enables detailed Kerberos login diagnostics. */ + public static final ConfigOption KERBEROS_DEBUG = + ConfigOptions.key("security.kerberos.login.debug") + .booleanType() + .defaultValue(false) + .withDescription("Enables Kerberos login diagnostics.") + .build(); + + /** Enables Kerberos authentication for Hadoop clients. */ + public static final ConfigOption KERBEROS_ENABLED = + ConfigOptions.key("security.kerberos.login.enable") + .booleanType() + .defaultValue(false) + .withDescription("Enables Kerberos authentication.") + .build(); + + /** Kerberos principal used for login. */ + public static final ConfigOption KERBEROS_PRINCIPAL = + ConfigOptions.key("security.kerberos.login.principal") + .stringType() + .defaultValue("") + .withDescription("Kerberos principal.") + .build(); + + /** Sensitive keytab path used for Kerberos login. */ + public static final ConfigOption KERBEROS_KEYTAB = + ConfigOptions.key("security.kerberos.login.keytab") + .stringType() + .defaultValue("") + .sensitive() + .withDescription("Kerberos keytab path.") + .build(); + + /** Path to the Kerberos client configuration file. */ + public static final ConfigOption KERBEROS_KRB5 = + ConfigOptions.key("security.kerberos.login.krb5") + .stringType() + .defaultValue("") + .withDescription("Kerberos krb5.conf path.") + .build(); + + /** Interval used to renew Kerberos login tickets. */ + public static final ConfigOption KERBEROS_TICKET_LIFETIME = + ConfigOptions.key("security.kerberos.ttl") + .durationType() + .defaultValue(Duration.ofHours(2)) + .check(value -> !value.isZero(), "ticket lifetime must be greater than zero") + .withDescription("Kerberos ticket renewal lifetime.") + .build(); + + private HadoopOptions() { + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/MavenOptions.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/MavenOptions.java new file mode 100644 index 0000000000..1e4b76fbbe --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/MavenOptions.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration.option; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; + +/** + * Maven resolver options used by built-in project compilation. + * + *

Authentication metadata is declared independently from resolver construction, and passwords + * are marked sensitive for configuration diagnostics. + */ +public final class MavenOptions { + + /** Absolute path to Maven's settings file. */ + public static final ConfigOption SETTINGS_PATH = + ConfigOptions.key("streampark.maven.settings") + .stringType() + .noDefaultValue() + .withDescription("Absolute path to Maven settings.xml.") + .build(); + + /** Remote Maven repository used by built-in compilation. */ + public static final ConfigOption REPOSITORY_URL = + ConfigOptions.key("streampark.maven.central.repository") + .stringType() + .defaultValue("https://repo1.maven.org/maven2/") + .withDescription("Maven repository used by built-in compilation.") + .build(); + + /** User name used to authenticate with the configured repository. */ + public static final ConfigOption USER_NAME = + ConfigOptions.key("streampark.maven.auth.user") + .stringType() + .noDefaultValue() + .withDescription("Maven repository user name.") + .build(); + + /** Sensitive password used to authenticate with the configured repository. */ + public static final ConfigOption PASSWORD = + ConfigOptions.key("streampark.maven.auth.password") + .stringType() + .noDefaultValue() + .sensitive() + .withDescription("Maven repository password.") + .build(); + + private MavenOptions() { + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/WorkspaceOptions.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/WorkspaceOptions.java new file mode 100644 index 0000000000..2c37e025de --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/WorkspaceOptions.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration.option; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; + +/** + * Local and remote workspace root options. + * + *

These declarations contain configured roots only. Storage-specific URI resolution and child + * path derivation are handled by {@link org.apache.streampark.common.configuration.Workspace}. + */ +public final class WorkspaceOptions { + + /** Root directory for artifacts stored on the local filesystem. */ + public static final ConfigOption LOCAL_ROOT = + ConfigOptions.key("streampark.workspace.local") + .stringType() + .defaultValue("/streampark") + .check(value -> !value.trim().isEmpty(), "workspace path must not be blank") + .withDescription("Root directory for local StreamPark artifacts.") + .build(); + + /** Root directory or HDFS URI for remotely persisted artifacts. */ + public static final ConfigOption REMOTE_ROOT = + ConfigOptions.key("streampark.workspace.remote") + .stringType() + .defaultValue("/streampark") + .check(value -> !value.trim().isEmpty(), "workspace path must not be blank") + .withDescription("Root directory for remote StreamPark artifacts.") + .build(); + + private WorkspaceOptions() { + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/YarnOptions.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/YarnOptions.java new file mode 100644 index 0000000000..168181e789 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/YarnOptions.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration.option; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; + +/** + * YARN ResourceManager connectivity and HTTP authentication options. + * + *

The authentication option is validated during typed access so unsupported mechanisms fail + * before an HTTP client is created. + */ +public final class YarnOptions { + + /** Optional ResourceManager proxy endpoint. */ + public static final ConfigOption PROXY_URL = + ConfigOptions.key("streampark.proxy.yarn-url") + .stringType() + .defaultValue("") + .withDescription("Optional YARN ResourceManager proxy URL.") + .build(); + + /** Authentication mechanism used for ResourceManager HTTP requests. */ + public static final ConfigOption HTTP_AUTHENTICATION = + ConfigOptions.key("streampark.yarn.http-auth") + .stringType() + .defaultValue("simple") + .check( + value -> "simple".equalsIgnoreCase(value) || "kerberos".equalsIgnoreCase(value), + "authentication must be 'simple' or 'kerberos'") + .withDescription("Authentication mechanism used for YARN HTTP requests.") + .build(); + + private YarnOptions() { + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/enums/TableMode.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/package-info.java similarity index 61% rename from streampark-common/src/main/java/org/apache/streampark/common/enums/TableMode.java rename to streampark-common/src/main/java/org/apache/streampark/common/configuration/option/package-info.java index 4262bc5e96..664ce20274 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/enums/TableMode.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/option/package-info.java @@ -15,24 +15,12 @@ * limitations under the License. */ -package org.apache.streampark.common.enums; - -import javax.annotation.Nonnull; - -/** Flink table execution mode. */ -public enum TableMode { - - BATCH, - STREAMING; - - /** - * Resolve enum constant by name (Scala {@code Enumeration.withName} compatibility). - * - * @param name enum constant name - * @return matching {@link TableMode} - */ - @Nonnull - public static TableMode withName(@Nonnull String name) { - return valueOf(name); - } -} +/** + * Engine-neutral typed option catalogs owned by {@code streampark-common}. + * + *

Catalog classes declare metadata only and must not read files, environment variables, system + * properties, or global configuration during class initialization. This keeps declarations safe + * to reuse in console, submission, and runtime modules. Engine-specific options belong to the + * corresponding engine module rather than this package. + */ +package org.apache.streampark.common.configuration.option; diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/package-info.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/package-info.java new file mode 100644 index 0000000000..264351b232 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/package-info.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * StreamPark's engine-neutral configuration model. + * + *

The package separates five responsibilities: + * + *

    + *
  1. {@link org.apache.streampark.common.configuration.ConfigOption} declares immutable typed + * metadata. + *
  2. {@link org.apache.streampark.common.configuration.ConfigurationParser} parses external + * documents without requiring an option registry. + *
  3. {@link org.apache.streampark.common.util.YamlParser} provides strict structured YAML + * parsing with a bounded compatibility fallback. + *
  4. {@link org.apache.streampark.common.configuration.ConfigurationLoader} composes named + * layers using the fixed source precedence. + *
  5. {@link org.apache.streampark.common.configuration.Configuration} exposes an immutable, + * origin-aware snapshot to consumers. + *
+ * + *

The lifecycle of a configuration value is declaration, parsing, layer composition, and typed + * access. Parsers deliberately retain raw scalar or collection values; conversion and validation + * happen only when a consumer reads a {@link + * org.apache.streampark.common.configuration.ConfigOption}. This permits independently deployed + * modules to define options without maintaining a central registry in {@code streampark-common}. + * + *

Precedence from lowest to highest is defaults, files, environment, JVM system properties, + * command line, and runtime overrides. Origins remain attached to effective values so conversion + * failures can identify the responsible source. Configuration parsing never mutates JVM system + * properties, and immutable snapshots prevent a running operation from observing partial updates. + * Engine-specific option catalogs belong to their engine module's {@code configuration} package. + */ +package org.apache.streampark.common.configuration; diff --git a/streampark-common/src/main/java/org/apache/streampark/common/constants/DateConstants.java b/streampark-common/src/main/java/org/apache/streampark/common/constants/DateConstants.java deleted file mode 100644 index ccb7277d10..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/constants/DateConstants.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.constants; - -public class DateConstants { - - /** - * date format of yyyy-MM-dd HH:mm:ss - */ - public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss"; - /** - * timestamp - */ - public static final String TIMESTAMP = "timestamp"; -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/conf/FlinkVersion.java b/streampark-common/src/main/java/org/apache/streampark/common/core/FlinkVersion.java similarity index 85% rename from streampark-common/src/main/java/org/apache/streampark/common/conf/FlinkVersion.java rename to streampark-common/src/main/java/org/apache/streampark/common/core/FlinkVersion.java index aa1fd6553f..509991f87f 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/conf/FlinkVersion.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/core/FlinkVersion.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.streampark.common.conf; +package org.apache.streampark.common.core; import org.apache.streampark.common.util.CommandUtils; +import org.apache.streampark.common.util.StreamParkLoggerFactory; import org.apache.streampark.shaded.org.slf4j.Logger; @@ -28,29 +29,37 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.function.Consumer; +import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -/** @param flinkHome actual flink home that must be a readable local path */ +/** + * Discovers Flink and Scala versions and runtime libraries from a local Flink installation. + * + *

Distribution JAR names are preferred because they avoid starting a separate Flink process. + * The command-line frontend is used only when the distribution name does not expose a version. + */ public class FlinkVersion implements Serializable { private static final long serialVersionUID = 1L; - private static final Logger LOG = - org.apache.streampark.common.util.StreamParkLoggerFactory.loggerFactory() - .getLogger(FlinkVersion.class.getName()); + + private static final Logger LOG = StreamParkLoggerFactory.loggerFactory().getLogger(FlinkVersion.class.getName()); private static final Pattern FLINK_VER_PATTERN = Pattern.compile("^(\\d+\\.\\d+)(?:\\..*)?$"); - private static final Pattern FLINK_VERSION_PATTERN = - Pattern.compile("^Version: ([^,]*), Commit ID: (.*)$"); + + private static final Pattern FLINK_VERSION_PATTERN = Pattern.compile("^Version: ([^,]*), Commit ID: (.*)$"); + private static final Pattern FLINK_DIST_UNDERSCORE_PATTERN = Pattern.compile( "^flink-dist_(\\d+\\.\\d+)-(\\d+\\.\\d+(?:\\.\\d+)?(?:-SNAPSHOT)?)\\.jar$"); + private static final Pattern FLINK_DIST_DASH_PATTERN = Pattern.compile("^flink-dist-(\\d+\\.\\d+(?:\\.\\d+)?(?:-SNAPSHOT)?)\\.jar$"); + private static final Pattern APACHE_FLINK_VERSION_PATTERN = Pattern.compile("^(\\d+\\.\\d+\\.\\d+)"); + private static final Pattern OTHER_FLINK_VERSION_PATTERN = Pattern.compile("^(\\d+\\.\\d+)-?$"); /** Flink installation directory (Scala {@code flinkHome} accessor). */ @@ -60,6 +69,11 @@ public class FlinkVersion implements Serializable { private transient File flinkLib; private transient File flinkDistJar; + /** + * Creates a lazily evaluated descriptor for a Flink installation. + * + * @param flinkHome readable local Flink installation directory + */ public FlinkVersion(String flinkHome) { this.flinkHome = flinkHome; } @@ -186,7 +200,7 @@ public boolean checkVersion(boolean throwException) { try { int major = Integer.parseInt(parts[0].trim()); int minor = Integer.parseInt(parts[1].trim()); - if (major == 1 && minor >= 17 && minor <= 20) { + if (major == 1 && minor >= 18 && minor <= 20) { return true; } if (major == 2 && minor >= 0 && minor <= 3) { @@ -219,7 +233,7 @@ public boolean checkVersion(int sinceVersion) { return false; } - private java.util.Optional parseFromDistJar() { + private Optional parseFromDistJar() { String jarName = getFlinkDistJar().getName(); Matcher underscoreMatcher = FLINK_DIST_UNDERSCORE_PATTERN.matcher(jarName); if (underscoreMatcher.matches()) { @@ -229,7 +243,7 @@ private java.util.Optional parseFromDistJar() { "Flink version parsed from dist jar name: {}, scala: {}", parsedVersion, parsedScala); - return java.util.Optional.of(new String[]{parsedVersion, parsedScala}); + return Optional.of(new String[]{parsedVersion, parsedScala}); } Matcher dashMatcher = FLINK_DIST_DASH_PATTERN.matcher(jarName); if (dashMatcher.matches()) { @@ -238,9 +252,9 @@ private java.util.Optional parseFromDistJar() { "Flink version parsed from dist jar name: {}, scala: {}", parsedVersion, "2.12"); - return java.util.Optional.of(new String[]{parsedVersion, "2.12"}); + return Optional.of(new String[]{parsedVersion, "2.12"}); } - return java.util.Optional.empty(); + return Optional.empty(); } private String parseFromCliFrontend() { @@ -255,22 +269,18 @@ private String parseFromCliFrontend() { CommandUtils.execute( getFlinkLib().getAbsolutePath(), cmd, - new Consumer() { - - @Override - public void accept(String out) { - buffer.append(out).append("\n"); - Matcher matcher = FLINK_VERSION_PATTERN.matcher(out); - if (matcher.find()) { - String ver = matcher.group(1); - Matcher m1 = APACHE_FLINK_VERSION_PATTERN.matcher(ver); - if (m1.find()) { + out -> { + buffer.append(out).append("\n"); + Matcher matcher = FLINK_VERSION_PATTERN.matcher(out); + if (matcher.find()) { + String ver = matcher.group(1); + Matcher m1 = APACHE_FLINK_VERSION_PATTERN.matcher(ver); + if (m1.find()) { + flinkVersion[0] = ver; + } else { + Matcher m2 = OTHER_FLINK_VERSION_PATTERN.matcher(ver); + if (m2.find()) { flinkVersion[0] = ver; - } else { - Matcher m2 = OTHER_FLINK_VERSION_PATTERN.matcher(ver); - if (m2.find()) { - flinkVersion[0] = ver; - } } } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/conf/SparkVersion.java b/streampark-common/src/main/java/org/apache/streampark/common/core/SparkVersion.java similarity index 95% rename from streampark-common/src/main/java/org/apache/streampark/common/conf/SparkVersion.java rename to streampark-common/src/main/java/org/apache/streampark/common/core/SparkVersion.java index f57aecd0ab..d344244ae2 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/conf/SparkVersion.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/core/SparkVersion.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.streampark.common.conf; +package org.apache.streampark.common.core; import org.apache.streampark.common.util.CommandUtils; +import org.apache.streampark.common.util.StreamParkLoggerFactory; import org.apache.streampark.shaded.org.slf4j.Logger; @@ -31,10 +32,13 @@ public class SparkVersion implements Serializable { private static final long serialVersionUID = 1L; - private static final Logger LOG = org.apache.streampark.common.util.StreamParkLoggerFactory.loggerFactory() - .getLogger(SparkVersion.class.getName()); + + private static final Logger LOG = StreamParkLoggerFactory.loggerFactory().getLogger(SparkVersion.class.getName()); + private static final Pattern SPARK_VER_PATTERN = Pattern.compile("^(\\d+\\.\\d+)(?:\\..*)?$"); + private static final Pattern SPARK_VERSION_PATTERN = Pattern.compile("\\s{2}version\\s(\\d+\\.\\d+\\.\\d+)"); + private static final Pattern SPARK_SCALA_VERSION_PATTERN = Pattern.compile("Using\\sScala\\sversion\\s(\\d+\\.\\d+)"); diff --git a/streampark-common/src/main/java/org/apache/streampark/common/enums/FlinkK8sRestExposedType.java b/streampark-common/src/main/java/org/apache/streampark/common/enums/FlinkKubernetesRestExposedType.java similarity index 83% rename from streampark-common/src/main/java/org/apache/streampark/common/enums/FlinkK8sRestExposedType.java rename to streampark-common/src/main/java/org/apache/streampark/common/enums/FlinkKubernetesRestExposedType.java index 231ef5f9da..7a9f729434 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/enums/FlinkK8sRestExposedType.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/enums/FlinkKubernetesRestExposedType.java @@ -21,7 +21,7 @@ import javax.annotation.Nullable; /** kubernetes.rest-service.exposed.type */ -public enum FlinkK8sRestExposedType { +public enum FlinkKubernetesRestExposedType { /** LoadBalancer */ LOAD_BALANCER("LoadBalancer", 0), @@ -36,18 +36,18 @@ public enum FlinkK8sRestExposedType { private final Integer type; - FlinkK8sRestExposedType(@Nonnull String name, @Nonnull Integer type) { + FlinkKubernetesRestExposedType(@Nonnull String name, @Nonnull Integer type) { this.name = name; this.type = type; } /** * Try to resolve the given Flink K8s Rest-Exposed type value into a known {@link - * FlinkK8sRestExposedType} enum. + * FlinkKubernetesRestExposedType} enum. */ @Nullable - public static FlinkK8sRestExposedType of(@Nullable Integer value) { - for (FlinkK8sRestExposedType order : values()) { + public static FlinkKubernetesRestExposedType of(@Nullable Integer value) { + for (FlinkKubernetesRestExposedType order : values()) { if (order.type.equals(value)) { return order; } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/enums/Semantic.java b/streampark-common/src/main/java/org/apache/streampark/common/enums/Semantic.java deleted file mode 100644 index 4d77eac861..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/enums/Semantic.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.enums; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; - -/** Flink consistency semantics */ -public enum Semantic { - - /** - * Ensure that the counting results obtained after a fault are consistent with the correct values. - */ - EXACTLY_ONCE, - - /** The program may calculate more after a malfunction, but it will never calculate less. */ - AT_LEAST_ONCE, - - /** After the fault occurs, the counting results may be lost. */ - NONE; - - /** Try to resolve the given semantic name into a known {@link Semantic}. */ - @Nullable - public static Semantic of(@Nonnull String name) { - for (Semantic semantic : Semantic.values()) { - if (name.equals(semantic.name())) { - return semantic; - } - } - return null; - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/ClassLoaderUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/ClassLoaderUtils.java index 020568e107..ab1eb254a5 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/ClassLoaderUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/ClassLoaderUtils.java @@ -17,7 +17,7 @@ package org.apache.streampark.common.util; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; import org.apache.streampark.shaded.org.slf4j.Logger; diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/ConfigUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/ConfigUtils.java deleted file mode 100644 index 3835d172f9..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/ConfigUtils.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.constants.Constants; - -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -public final class ConfigUtils { - - private ConfigUtils() { - } - - public static Properties getConf(Map parameter) { - return getConf(parameter, "", "", ""); - } - - public static Properties getConf(Map parameter, String prefix) { - return getConf(parameter, prefix, "", ""); - } - - public static Properties getConf( - Map parameter, String prefix, String addfix, String alias) { - Map map = filterParam(parameter, prefix + alias); - Properties prop = new Properties(); - for (Map.Entry entry : map.entrySet()) { - if (entry.getValue() != null && !entry.getValue().isEmpty()) { - prop.put(addfix + entry.getKey(), entry.getValue()); - } - } - return prop; - } - - public static Properties getHBaseConfig(Map parameter) { - return getConf(parameter, ConfigKeys.HBASE_PREFIX(), ConfigKeys.HBASE_PREFIX(), ""); - } - - public static Properties getInfluxConfig(Map parameter) { - return getConf(parameter, ConfigKeys.INFLUX_PREFIX(), "", ""); - } - - public static Properties getKafkaSinkConf(Map parameter, String topic) { - return getKafkaSinkConf(parameter, topic, ""); - } - - public static Properties getKafkaSinkConf( - Map parameter, String topic, String alias) { - String prefix = ConfigKeys.KAFKA_SINK_PREFIX() + alias; - if (!prefix.endsWith(".")) { - prefix = prefix + "."; - } - Map param = filterParam(parameter, prefix); - if (param.isEmpty()) { - throw new IllegalArgumentException(topic + " init error..."); - } - Properties kafkaProperty = new Properties(); - for (Map.Entry entry : param.entrySet()) { - kafkaProperty.put(entry.getKey(), entry.getValue().trim()); - } - String resolvedTopic = topic; - if (Constants.EMPTY_STRING.equals(topic)) { - Object top = kafkaProperty.get(ConfigKeys.KEY_KAFKA_TOPIC()); - if (top == null || top.toString().split(",|\\s+").length > 1) { - throw new IllegalArgumentException( - "Can't find a unique topic!!!,you must be input a topic"); - } - resolvedTopic = top.toString(); - } - boolean hasTopic = true; - for (Map.Entry entry : kafkaProperty.entrySet()) { - if (ConfigKeys.KEY_KAFKA_TOPIC().equals(entry.getKey())) { - for (String t : entry.getValue().toString().split(",|\\s+")) { - if (t.equals(resolvedTopic)) { - hasTopic = false; - break; - } - } - } - } - if (hasTopic) { - throw new IllegalArgumentException("Can't find a topic of:" + resolvedTopic + "!!!"); - } - kafkaProperty.put(ConfigKeys.KEY_KAFKA_TOPIC(), resolvedTopic); - return kafkaProperty; - } - - public static Properties getJdbcConf(Map parameter) { - return getJdbcConf(parameter, ""); - } - - public static Properties getJdbcConf(Map parameter, String alias) { - String prefix; - if (alias == null || alias.isEmpty()) { - prefix = ConfigKeys.KEY_JDBC_PREFIX(); - } else { - prefix = ConfigKeys.KEY_JDBC_PREFIX() + alias; - if (!prefix.endsWith(".")) { - prefix = prefix + "."; - } - } - String driver = parameter.getOrDefault(prefix + ConfigKeys.KEY_JDBC_DRIVER(), null); - String url = parameter.getOrDefault(prefix + ConfigKeys.KEY_JDBC_URL(), null); - String user = parameter.getOrDefault(prefix + ConfigKeys.KEY_JDBC_USER(), null); - String password = parameter.getOrDefault(prefix + ConfigKeys.KEY_JDBC_PASSWORD(), null); - if (driver == null || url == null) { - throw new IllegalArgumentException( - "Jdbc instance:" + prefix + " error,[driver|url] must not be null"); - } - if ((user != null && password == null) || (user == null && password != null)) { - throw new IllegalArgumentException( - "Jdbc instance:" - + prefix - + " error, [user|password] must be all null,or all not null "); - } - Map param = filterParam(parameter, prefix); - Properties properties = new Properties(); - String aliasName = alias == null || alias.trim().isEmpty() ? "default" : alias; - properties.put(ConfigKeys.KEY_ALIAS(), aliasName); - properties.put(ConfigKeys.KEY_JDBC_DRIVER(), driver); - for (Map.Entry entry : param.entrySet()) { - properties.put(entry.getKey(), entry.getValue()); - } - return properties; - } - - public static Properties getConf(Properties properties, String prefix, String addfix) { - return getConfFromProperties(properties, prefix, addfix, ""); - } - - public static Properties getConfFromProperties( - Properties properties, String prefix, String addfix, String alias) { - Map map = new HashMap<>(); - if (properties != null) { - for (Map.Entry entry : properties.entrySet()) { - if (entry.getKey() != null && entry.getValue() != null) { - map.put(entry.getKey().toString(), entry.getValue().toString()); - } - } - } - return getConf(map, prefix, addfix, alias); - } - - public static Map getConfMap(Properties properties, String prefix) { - Properties props = getConfFromProperties(properties, prefix, "", ""); - Map result = new HashMap<>(); - for (Map.Entry entry : props.entrySet()) { - result.put(entry.getKey().toString(), entry.getValue().toString()); - } - return result; - } - - private static Map filterParam(Map parameter, String fix) { - Map result = new HashMap<>(); - for (Map.Entry entry : parameter.entrySet()) { - if (entry.getKey().startsWith(fix) && entry.getValue() != null) { - result.put( - entry.getKey().substring(fix.length()).replaceFirst("^\\.", ""), - entry.getValue()); - } - } - return result; - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/DeflaterUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/DeflaterUtils.java index 8fb587c90a..4bc5afbad7 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/DeflaterUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/DeflaterUtils.java @@ -17,11 +17,8 @@ package org.apache.streampark.common.util; -import org.apache.streampark.shaded.org.slf4j.Logger; - -import org.apache.commons.lang3.StringUtils; - import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.zip.DataFormatException; import java.util.zip.Deflater; @@ -29,72 +26,99 @@ public final class DeflaterUtils { - private static final Logger LOG = - StreamParkLoggerFactory.loggerFactory().getLogger(DeflaterUtils.class.getName()); + private static final int BUFFER_SIZE = 256; + private static final int MAX_NESTED_COMPRESSION_DEPTH = 16; private DeflaterUtils() { } + /** Compresses UTF-8 text and returns its Base64 representation. */ public static String zipString(String text) { - if (StringUtils.isBlank(text)) { + if (text == null || text.isEmpty()) { return ""; } Deflater deflater = new Deflater(Deflater.BEST_COMPRESSION); - deflater.setInput(text.getBytes()); - deflater.finish(); - byte[] bytes = new byte[256]; - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(256); - while (!deflater.finished()) { - int length = deflater.deflate(bytes); - outputStream.write(bytes, 0, length); + try { + deflater.setInput(text.getBytes(StandardCharsets.UTF_8)); + deflater.finish(); + byte[] buffer = new byte[BUFFER_SIZE]; + ByteArrayOutputStream output = new ByteArrayOutputStream(BUFFER_SIZE); + while (!deflater.finished()) { + int length = deflater.deflate(buffer); + output.write(buffer, 0, length); + } + return Base64.getEncoder().encodeToString(output.toByteArray()); + } finally { + deflater.end(); } - deflater.end(); - return Base64.getEncoder().encodeToString(outputStream.toByteArray()); } - public static String unzipString(String zipString) { - byte[] decode; + /** Decompresses Base64-encoded UTF-8 text, or returns {@code null} for invalid input. */ + public static String unzipString(String compressedText) { + if (compressedText == null) { + return null; + } + if (compressedText.isEmpty()) { + return ""; + } + + byte[] compressedBytes; try { - decode = Base64.getDecoder().decode(zipString); + compressedBytes = Base64.getDecoder().decode(compressedText); } catch (IllegalArgumentException e) { return null; } + Inflater inflater = new Inflater(); - inflater.setInput(decode); - byte[] bytes = new byte[256]; - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(256); + inflater.setInput(compressedBytes); + byte[] buffer = new byte[BUFFER_SIZE]; + ByteArrayOutputStream output = new ByteArrayOutputStream(BUFFER_SIZE); try { while (!inflater.finished()) { - int length = inflater.inflate(bytes); - outputStream.write(bytes, 0, length); + int length = inflater.inflate(buffer); + if (length > 0) { + output.write(buffer, 0, length); + continue; + } + if (inflater.finished()) { + break; + } + if (inflater.needsInput() || inflater.needsDictionary()) { + return null; + } + // Inflater made no progress and cannot request more input. Treat it as corrupt. + return null; } } catch (DataFormatException e) { - LOG.warn("Failed to unzip string", e); return null; } finally { inflater.end(); } - return outputStream.toString(); + return new String(output.toByteArray(), StandardCharsets.UTF_8); } - /** Returns plain text, repeatedly decoding when the input is compressed multiple times. */ + /** + * Returns plain text while accepting legacy values that were stored without compression or + * compressed more than once. + */ public static String toPlainText(String text) { - if (StringUtils.isBlank(text)) { + if (text == null || text.isEmpty()) { return text; } String current = text; - while (true) { + for (int depth = 0; depth < MAX_NESTED_COMPRESSION_DEPTH; depth++) { String decoded = unzipString(current); - if (decoded == null) { + if (decoded == null || decoded.equals(current)) { return current; } current = decoded; } + return current; } - /** Stores SQL/conf text as a single compressed blob regardless of input encoding. */ + /** Normalizes plain or legacy nested-compressed text to one compressed representation. */ public static String compressForStorage(String text) { - if (StringUtils.isBlank(text)) { + if (text == null || text.isEmpty()) { return ""; } return zipString(toPlainText(text)); diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java index 44b5f6445b..62266bb987 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java @@ -256,7 +256,7 @@ public static void readInputStream(InputStream in, byte[] array) throws IOExcept } public static String readFile(String filename) throws IOException { - return SafePathUtils.readConfigFile(filename); + return PathUtils.readFileAsString(filename); } public static String readFile(File file) throws IOException { diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java index 91e55d0340..fca6768359 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java @@ -17,220 +17,345 @@ package org.apache.streampark.common.util; -import org.apache.streampark.shaded.org.slf4j.Logger; - -import org.apache.commons.lang3.StringUtils; +import org.apache.streampark.common.configuration.ConfigException; +import org.apache.streampark.common.configuration.SensitiveKeys; +import org.apache.streampark.common.core.FlinkVersion; -import javax.annotation.Nonnull; +import org.apache.streampark.shaded.org.slf4j.Logger; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.Objects; import java.util.regex.Matcher; import java.util.regex.Pattern; -/** Flink configuration parsing utilities. */ +/** + * Utilities for loading native Flink configuration across legacy and standard YAML formats. + * + *

Flink versions before 1.19 require the flat {@code flink-conf.yaml} format. Flink 1.19 and + * 1.20 accept both filenames, prefer {@code flink-conf.yaml} when both exist, and select the parser + * from the chosen filename. Flink 2.0 removes the legacy filename and requires standard YAML in + * {@code config.yaml}. Keeping file selection and parser selection together prevents the + * StreamPark classpath version from changing target-runtime behavior. + * + *

Directory-based loading always requires the target Flink version. Callers that already know + * the selected file or transported YAML format can use the corresponding explicit entry point. + */ public final class FlinkConfigurationUtils { + /** Filename used by Flink's historical flat configuration syntax. */ + public static final String LEGACY_FLINK_CONF_FILENAME = "flink-conf.yaml"; + + /** Filename used by Flink's standard nested YAML syntax. */ + public static final String FLINK_CONF_FILENAME = "config.yaml"; + + private static final int DUAL_FORMAT_MAJOR = 1; + private static final int DUAL_FORMAT_MINOR = 19; + private static final int STANDARD_ONLY_MAJOR = 2; + private static final int STANDARD_ONLY_MINOR = 0; + private static final Pattern FLINK_VERSION_PATTERN = + Pattern.compile("^(\\d+)\\.(\\d+)(?:\\..*)?$"); + private static final Logger LOG = StreamParkLoggerFactory.loggerFactory() .getLogger(FlinkConfigurationUtils.class.getName()); - private static final Pattern PROPERTY_PATTERN = Pattern.compile("(.*?)=(.*?)"); + private FlinkConfigurationUtils() { + } - private static final Pattern MULTI_PROPERTY_PATTERN = - Pattern.compile("-D(.*?)\\s*=\\s*([\"'])(.*)\\2"); + /** + * Loads configuration from a Flink installation using the version discovered from its dist + * jar. + * + * @param flinkHome Flink installation directory + * @return loaded Flink configuration entries + * @throws ConfigException when the installation or its required configuration is invalid + */ + public static Map loadFlinkHome(String flinkHome) { + String home = Objects.requireNonNull(flinkHome, "Flink home must not be null"); + FlinkVersion flinkVersion = new FlinkVersion(home); + return loadConfiguration( + Path.of(home, "conf").toString(), flinkVersion.version()); + } - private FlinkConfigurationUtils() { + /** + * Loads the configuration file required by a concrete Flink version. + * + *

Flink 1.19 and 1.20 prefer {@code flink-conf.yaml}, with {@code config.yaml} as fallback. + * Earlier releases require the legacy file, while Flink 2.0 and later require standard YAML. + * + * @param configDirectory Flink configuration directory + * @param flinkVersion target Flink version in major-minor or major-minor-patch form + * @return loaded Flink configuration entries + * @throws ConfigException when the required file is absent or parsing fails + */ + public static Map loadConfiguration( + String configDirectory, + String flinkVersion) { + return loadConfigurationFromFile( + resolveConfigurationFile(configDirectory, flinkVersion)); } - public static Map loadFlinkConf(File file) { - AssertUtils.required( - file != null && file.exists() && file.isFile(), - "[StreamPark] loadFlinkConfYaml: file must not be null"); - try { - return loadFlinkConf( - org.apache.commons.io.FileUtils.readFileToString(file, StandardCharsets.UTF_8)); - } catch (IOException e) { - throw new IllegalStateException("Failed to read Flink configuration file: " + file, e); + /** + * Loads a single Flink YAML file, selecting legacy behavior by its filename. + * + *

The exact filename {@code flink-conf.yaml} uses Flink's historical line parser, while + * {@code config.yaml} uses the standard YAML parser. Other filenames are rejected because the + * parser cannot be inferred safely from content. + * + * @param file existing Flink YAML file + * @return loaded Flink configuration entries + * @throws ConfigException when the file is absent or invalid + */ + public static Map loadConfigurationFromFile(File file) { + if (file == null || !file.isFile()) { + throw new ConfigException("Flink configuration file does not exist: " + file); } + boolean standardYaml = isStandardYaml(file); + LOG.info( + "Loading {} Flink YAML configuration from {}", + standardYaml ? "standard" : "legacy", + file); + Path path = file.toPath(); + Map configuration = standardYaml ? loadStandard(path) : loadLegacy(path); + logConfiguration("Loaded", configuration); + return configuration; } - public static Map loadFlinkConf(String yaml) { - AssertUtils.required( - yaml != null && !yaml.isEmpty(), "[StreamPark] loadFlinkConfYaml: yaml must not be null"); - return new HashMap<>(PropertiesUtils.fromYamlText(yaml)); + /** + * Returns the parser mode defined by a supported Flink configuration filename. + * + * @param file Flink configuration file + * @return {@code true} for {@code config.yaml}, {@code false} for {@code flink-conf.yaml} + * @throws ConfigException when the filename is not supported + */ + public static boolean isStandardYaml(File file) { + if (file == null) { + throw new IllegalArgumentException("Flink configuration file must not be null"); + } + if (FLINK_CONF_FILENAME.equals(file.getName())) { + return true; + } + if (LEGACY_FLINK_CONF_FILENAME.equals(file.getName())) { + return false; + } + throw new ConfigException("Unsupported Flink configuration filename: " + file.getName()); } - public static Map loadLegacyFlinkConf(File file) { - AssertUtils.required( - file != null && file.exists() && file.isFile(), - "[StreamPark] loadFlinkConfYaml: file must not be null"); - try { - return loadLegacyFlinkConf( - org.apache.commons.io.FileUtils.readFileToString(file, StandardCharsets.UTF_8)); - } catch (IOException e) { - throw new IllegalStateException("Failed to read legacy Flink configuration file: " + file, e); - } - } - - public static Map loadLegacyFlinkConf(String yaml) { - AssertUtils.required( - yaml != null && !yaml.isEmpty(), "[StreamPark] loadFlinkConfYaml: yaml must not be null"); - Map flinkConf = new LinkedHashMap<>(); - java.util.Scanner scanner = new java.util.Scanner(yaml); - AtomicInteger lineNo = new AtomicInteger(0); - while (scanner.hasNextLine()) { - String line = scanner.nextLine(); - lineNo.incrementAndGet(); - int commentStart = line.indexOf('#'); - String conf = (commentStart >= 0 ? line.substring(0, commentStart) : line).trim(); - if (!conf.isEmpty()) { - String[] kv = conf.split(": ", 2); - if (kv.length == 2) { - String key = kv[0].trim(); - String value = kv[1].trim(); - if (!key.isEmpty() && !value.isEmpty()) { - flinkConf.put(key, value); - } else { - LOG.warn( - "[StreamPark] Error after splitting key and value in configuration {}: {}", - lineNo.get(), - line); - } - } else { - LOG.warn( - "[StreamPark] Error while trying to split key and value in configuration. {} : {}", - lineNo.get(), - line); - } - } + /** + * Resolves the configuration file for a Flink installation without conflating version and YAML + * syntax compatibility. + * + *

The transition rule mirrors Flink's rollout of standard YAML: + * + *

    + *
  • Flink before 1.19: {@code flink-conf.yaml} + *
  • Flink 1.19 and 1.20: prefer {@code flink-conf.yaml}, otherwise {@code config.yaml} + *
  • Flink 2.0 and later: {@code config.yaml} + *
+ * + * @param configDirectory Flink configuration directory + * @param flinkVersion target Flink version + * @return selected configuration file + * @throws ConfigException when the version is invalid or the required file is absent + */ + public static File resolveConfigurationFile( + String configDirectory, + String flinkVersion) { + if (configDirectory == null) { + throw new IllegalArgumentException("Flink configuration directory must not be null"); } - scanner.close(); - return flinkConf; - } - - @Nonnull - public static Map extractDynamicProperties(String properties) { - if (StringUtils.isEmpty(properties)) { - return Collections.emptyMap(); - } - Map map = new LinkedHashMap<>(); - String simple = MULTI_PROPERTY_PATTERN.matcher(properties).replaceAll(""); - String[] parts = simple.split("\\s?-D"); - if (Utils.isNotEmpty(parts)) { - for (String x : parts) { - if (!x.isEmpty()) { - Matcher p = PROPERTY_PATTERN.matcher(x.trim()); - if (p.matches()) { - map.put(p.group(1).trim(), p.group(2).trim()); - } - } - } + Path directory = requireDirectory(configDirectory); + if (compareVersion(flinkVersion, DUAL_FORMAT_MAJOR, DUAL_FORMAT_MINOR) < 0) { + return requireFile(directory, LEGACY_FLINK_CONF_FILENAME, flinkVersion); } - Matcher matcher = MULTI_PROPERTY_PATTERN.matcher(properties); - while (matcher.find()) { - map.put(matcher.group(1).trim(), matcher.group(3)); - } - return map; - } - - @Nonnull - public static List extractArguments(String args) { - if (StringUtils.isNotEmpty(args)) { - return extractArguments(args.split("\\s+")); - } - return new ArrayList<>(); - } - - public static List extractArguments(String[] array) { - List programArgs = new ArrayList<>(); - Iterator iter = Arrays.asList(array).iterator(); - while (iter.hasNext()) { - String v = iter.next(); - String p = v.substring(0, 1); - if ("'".equals(p) || "\"".equals(p)) { - String value = v; - if (!v.endsWith(p)) { - while (!value.endsWith(p) && iter.hasNext()) { - value += " " + iter.next(); - } - } - programArgs.add(value.substring(1, value.length() - 1)); - } else { - int eqIndex = v.indexOf('='); - if (eqIndex > 0 && v.length() > 2) { - char open = v.charAt(0); - char close = v.charAt(v.length() - 1); - if ((open == '\'' && close == '\'') || (open == '"' && close == '"')) { - programArgs.add(v.substring(1, v.length() - 1)); - } else if (eqIndex > 1 && v.charAt(eqIndex - 1) == '\'' && v.endsWith("'")) { - programArgs.add(v.substring(0, eqIndex - 1) + "=" + v.substring(eqIndex + 1, v.length() - 1)); - } else if (eqIndex > 1 && v.charAt(eqIndex - 1) == '"' && v.endsWith("\"")) { - programArgs.add(v.substring(0, eqIndex - 1) + "=" + v.substring(eqIndex + 1, v.length() - 1)); - } else { - programArgs.add(v); - } - } else { - programArgs.add(v); - } + if (compareVersion(flinkVersion, STANDARD_ONLY_MAJOR, STANDARD_ONLY_MINOR) < 0) { + Path legacyFile = directory.resolve(LEGACY_FLINK_CONF_FILENAME); + if (Files.isRegularFile(legacyFile)) { + return legacyFile.toFile(); } } - return programArgs; - } - - public static Map> extractMultipleArguments(String[] array) { - Iterator iter = Arrays.asList(array).iterator(); - Map> map = new LinkedHashMap<>(); - while (iter.hasNext()) { - String v = iter.next(); - if (v.length() >= 2 && v.startsWith("--")) { - String kv = iter.next(); - int eqIndex = kv.indexOf('='); - if (eqIndex > 0) { - String[] values = new String[]{kv.substring(0, eqIndex), kv.substring(eqIndex + 1)}; - String k1 = values[0].trim(); - String v1 = stripOuterQuotes(values[1].trim()); - String k = v.substring(2); - map.computeIfAbsent(k, key -> new LinkedHashMap<>()).put(k1, v1); - } + return requireFile(directory, FLINK_CONF_FILENAME, flinkVersion); + } + + /** + * Returns whether the selected configuration uses standard YAML. + * + *

Flink 1.19 and 1.20 select the mode from the preferred existing filename. Other releases + * have an unambiguous format contract. + * + * @param configDirectory Flink configuration directory + * @param flinkVersion target Flink version + * @return {@code true} when the selected file must use standard YAML semantics + * @throws ConfigException when the version is invalid + */ + public static boolean usesStandardYaml( + String configDirectory, + String flinkVersion) { + if (configDirectory == null) { + throw new IllegalArgumentException("Flink configuration directory must not be null"); + } + return isStandardYaml(resolveConfigurationFile(configDirectory, flinkVersion)); + } + + /** + * Parses standard, nested Flink YAML text. + * + * @param yaml non-blank standard YAML content + * @return loaded Flink configuration entries + * @throws ConfigException when the document is empty or invalid + */ + public static Map loadConfigurationFromString(String yaml) { + return loadConfigurationFromString(yaml, true); + } + + /** + * Parses Flink YAML text using explicit standard or legacy semantics. + * + *

The format flag is required when configuration has been compressed or transported without + * its filename. Guessing from document content is deliberately avoided because valid scalar + * values can be interpreted differently by the two parsers. + * + * @param yaml non-blank YAML content + * @param standardYaml whether to use the standard structured YAML parser + * @return loaded Flink configuration entries + * @throws ConfigException when the document is empty or invalid + */ + public static Map loadConfigurationFromString( + String yaml, + boolean standardYaml) { + if (yaml == null || yaml.trim().isEmpty()) { + throw new ConfigException("Flink YAML configuration must not be empty"); + } + return standardYaml ? loadStandard(yaml) : loadLegacy(yaml); + } + + private static Map loadStandard(Path path) { + return toConfigurationMap(YamlParser.flatten(YamlParser.parse(path))); + } + + private static Map loadStandard(String yaml) { + return toConfigurationMap(YamlParser.flatten(YamlParser.parse(yaml))); + } + + private static Map toConfigurationMap(Map values) { + Map configuration = new LinkedHashMap<>(); + // Standard YAML permits nested mappings and sequences. Mappings have already been + // flattened; lists retain YAML flow syntax because Flink expects values as strings. + values.forEach( + (key, value) -> configuration.put( + key, + value instanceof List + ? YamlParser.toFlowString(value) + : Objects.toString(value))); + return configuration; + } + + private static Map loadLegacy(Path path) { + try (BufferedReader reader = Files.newBufferedReader(path, StandardCharsets.UTF_8)) { + return loadLegacy(reader, path.toAbsolutePath().normalize().toString()); + } catch (IOException e) { + throw new ConfigException("Cannot read legacy Flink configuration from " + path, e); + } + } + + private static Map loadLegacy(String yaml) { + try (BufferedReader reader = new BufferedReader(new java.io.StringReader(yaml))) { + return loadLegacy(reader, "in-memory legacy Flink YAML"); + } catch (IOException e) { + throw new ConfigException("Cannot read in-memory legacy Flink configuration", e); + } + } + + /** + * Reproduces Flink's historical line-oriented format. Invalid lines are ignored with a warning + * because older Flink releases use the same compatibility behavior during startup. + */ + private static Map loadLegacy(BufferedReader reader, String resourceName) throws IOException { + Map configuration = new LinkedHashMap<>(); + String line; + int lineNumber = 0; + while ((line = reader.readLine()) != null) { + lineNumber++; + String content = line.split("#", 2)[0].trim(); + if (content.isEmpty()) { + continue; + } + String[] entry = content.split(": ", 2); + if (entry.length != 2) { + LOG.warn( + "Ignoring invalid Flink configuration line {}:{}; expected 'key: value'", + resourceName, + lineNumber); + continue; + } + String key = entry[0].trim(); + String value = entry[1].trim(); + if (key.isEmpty() || value.isEmpty()) { + LOG.warn( + "Ignoring empty Flink configuration entry at {}:{}", + resourceName, + lineNumber); + continue; } + configuration.put(key, value); } - return map; + return configuration; } - @Nonnull - public static Map extractDynamicPropertiesAsJava(String properties) { - return new HashMap<>(extractDynamicProperties(properties)); + private static void logConfiguration(String action, Map configuration) { + LOG.info("{} {} Flink configuration properties", action, configuration.size()); + configuration.forEach( + (key, value) -> LOG.debug( + "{} Flink configuration property: {}={}", + action, + key, + SensitiveKeys.isSensitive(key) ? SensitiveKeys.MASK : value)); + } + + private static Path requireDirectory(String configDirectory) { + Path directory = Path.of(configDirectory); + if (!Files.isDirectory(directory)) { + throw new ConfigException( + "Flink configuration directory does not exist: " + + directory.toAbsolutePath().normalize()); + } + return directory; } - @Nonnull - public static Map> extractMultipleArgumentsAsJava(String[] args) { - Map> result = new HashMap<>(); - for (Map.Entry> entry : extractMultipleArguments(args).entrySet()) { - result.put(entry.getKey(), new HashMap<>(entry.getValue())); + private static File requireFile(Path directory, String filename, String flinkVersion) { + Path selected = directory.resolve(filename); + if (!Files.isRegularFile(selected)) { + throw new ConfigException( + "Flink " + + flinkVersion + + " configuration file does not exist: " + + selected.toAbsolutePath().normalize()); } - return result; + return selected.toFile(); } - private static String stripOuterQuotes(String value) { - if (value == null || value.length() < 2) { - return value; + private static int compareVersion(String flinkVersion, int targetMajor, int targetMinor) { + if (flinkVersion == null) { + throw new IllegalArgumentException("Flink version must not be null"); } - char first = value.charAt(0); - char last = value.charAt(value.length() - 1); - if ((first == '"' && last == '"') || (first == '\'' && last == '\'')) { - return value.substring(1, value.length() - 1); + Matcher matcher = FLINK_VERSION_PATTERN.matcher(flinkVersion.trim()); + if (!matcher.matches()) { + throw new ConfigException("Invalid Flink version: " + flinkVersion); } - return value; + int major = Integer.parseInt(matcher.group(1)); + int minor = Integer.parseInt(matcher.group(2)); + // Decimal parsing would incorrectly place version 1.20 before version 1.19. + int majorComparison = Integer.compare(major, targetMajor); + return majorComparison != 0 + ? majorComparison + : Integer.compare(minor, targetMinor); } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/HBaseClient.java b/streampark-common/src/main/java/org/apache/streampark/common/util/HBaseClient.java deleted file mode 100644 index 2232dc49a6..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/HBaseClient.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -import org.apache.streampark.common.conf.ConfigKeys; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.ConnectionFactory; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.security.UserGroupInformation; - -import java.io.Serializable; -import java.util.Properties; -import java.util.function.Supplier; - -public class HBaseClient implements Serializable { - - private static final long serialVersionUID = 1L; - public static final Configuration CONF = HBaseConfiguration.create(); - private final Supplier func; - private transient Connection connection; - - public HBaseClient(Supplier func) { - this.func = func; - } - - public Connection getConnection() { - if (connection == null) - connection = func.get(); - return connection; - } - - public Table table(String table) throws Exception { - return getConnection().getTable(TableName.valueOf(table)); - } - - public static HBaseClient apply(Properties prop) { - Object user = prop.remove(ConfigKeys.KEY_HBASE_AUTH_USER()); - for (String key : prop.stringPropertyNames()) - CONF.set(key, prop.getProperty(key)); - return new HBaseClient(() -> { - try { - if (user != null) { - UserGroupInformation.setConfiguration(CONF); - UserGroupInformation remoteUser = UserGroupInformation.createRemoteUser(user.toString()); - UserGroupInformation.setLoginUser(remoteUser); - } - Connection connection = ConnectionFactory.createConnection(CONF); - Runtime.getRuntime() - .addShutdownHook( - new Thread( - () -> { - try { - connection.close(); - } catch (Exception ignored) { - // ignore - } - })); - return connection; - } catch (Exception e) { - throw new IllegalStateException("Failed to create HBase connection", e); - } - }); - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopConfigUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopConfigUtils.java index 409a5b13d7..9404e0f579 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopConfigUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopConfigUtils.java @@ -17,9 +17,8 @@ package org.apache.streampark.common.util; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.conf.InternalConfigHolder; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.option.HadoopOptions; import org.apache.streampark.common.fs.LfsOperator; import java.io.File; @@ -27,12 +26,10 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.Properties; import scala.Option; @@ -45,43 +42,37 @@ public final class HadoopConfigUtils { private static final String[] HIVE_CLIENT_CONF_FILES = {"core-site.xml", "hdfs-site.xml", "hive-site.xml"}; - private static final Map KERBEROS_CONF = loadKerberosConf(); - - public static final String HADOOP_USER_NAME = - InternalConfigHolder.get(CommonConfig.STREAMPARK_HADOOP_USER_NAME()); - + /** Returns the Hadoop user from the current configuration snapshot. */ public static String hadoopUserName() { - return HADOOP_USER_NAME; + return GlobalConfiguration.current().get(HadoopOptions.USER_NAME); } - public static final String KERBEROS_DEBUG = - KERBEROS_CONF.getOrDefault(ConfigKeys.KEY_SECURITY_KERBEROS_DEBUG(), "false"); - - public static final boolean KERBEROS_ENABLE = - Boolean.parseBoolean( - KERBEROS_CONF.getOrDefault(ConfigKeys.KEY_SECURITY_KERBEROS_ENABLE(), "false")); + /** Returns whether Kerberos authentication is enabled. */ + public static boolean kerberosEnabled() { + return GlobalConfiguration.current().get(HadoopOptions.KERBEROS_ENABLED); + } - public static final String KERBEROS_PRINCIPAL = - KERBEROS_CONF.getOrDefault(ConfigKeys.KEY_SECURITY_KERBEROS_PRINCIPAL(), "").trim(); + /** Returns whether Kerberos login diagnostics are enabled. */ + public static boolean kerberosDebug() { + return GlobalConfiguration.current().get(HadoopOptions.KERBEROS_DEBUG); + } - public static final String KERBEROS_KEYTAB = - KERBEROS_CONF.getOrDefault(ConfigKeys.KEY_SECURITY_KERBEROS_KEYTAB(), "").trim(); + /** Returns the configured Kerberos principal. */ + public static String kerberosPrincipal() { + return GlobalConfiguration.current().get(HadoopOptions.KERBEROS_PRINCIPAL).trim(); + } - public static final String KERBEROS_KRB5 = - KERBEROS_CONF.getOrDefault(ConfigKeys.KEY_SECURITY_KERBEROS_KRB5_CONF(), ""); + /** Returns the configured Kerberos keytab path. */ + public static String kerberosKeytab() { + return GlobalConfiguration.current().get(HadoopOptions.KERBEROS_KEYTAB).trim(); + } - private HadoopConfigUtils() { + /** Returns the configured krb5.conf path. */ + public static String kerberosKrb5() { + return GlobalConfiguration.current().get(HadoopOptions.KERBEROS_KRB5).trim(); } - private static Map loadKerberosConf() { - Properties props = System.getProperties(); - Map map = new HashMap<>(); - for (String key : props.stringPropertyNames()) { - if (key.startsWith("security.kerberos")) { - map.put(key, props.getProperty(key)); - } - } - return map; + private HadoopConfigUtils() { } public static Option getSystemHadoopConfDir() { diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopUtils.java index 6e2ba78c2c..05b5af6263 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/HadoopUtils.java @@ -17,9 +17,8 @@ package org.apache.streampark.common.util; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.conf.InternalConfigHolder; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.option.HadoopOptions; import org.apache.streampark.shaded.org.slf4j.Logger; @@ -77,10 +76,10 @@ private HadoopUtils() { public static UserGroupInformation getUgi() { if (ugi == null) { - if (HadoopConfigUtils.KERBEROS_ENABLE) { + if (HadoopConfigUtils.kerberosEnabled()) { ugi = getKerberosUGI(); } else { - ugi = UserGroupInformation.createRemoteUser(HadoopConfigUtils.HADOOP_USER_NAME); + ugi = UserGroupInformation.createRemoteUser(HadoopConfigUtils.hadoopUserName()); } } return ugi; @@ -115,27 +114,10 @@ private static long getTgtRefreshTime() { tgtRefreshTime = (long) ((end - start) * 0.90f); } else { LOG.warn("[StreamPark] get kerberos tgtRefreshTime failed, try get kerberos.ttl."); - DateUtils.TimeUnitPair timeUnit = - DateUtils.getTimeUnit(InternalConfigHolder.get(CommonConfig.KERBEROS_TTL())); - switch (timeUnit.unit) { - case SECONDS: - tgtRefreshTime = (long) timeUnit.num * 1000; - break; - case MINUTES: - tgtRefreshTime = (long) timeUnit.num * 60 * 1000; - break; - case HOURS: - tgtRefreshTime = (long) timeUnit.num * 60 * 60 * 1000; - break; - case DAYS: - tgtRefreshTime = (long) timeUnit.num * 60 * 60 * 24 * 1000; - break; - default: - throw new IllegalArgumentException( - "[StreamPark] parameter:" - + CommonConfig.KERBEROS_TTL().getKey() - + " invalided, unit options are [s|m|h|d]"); - } + tgtRefreshTime = + GlobalConfiguration.current() + .get(HadoopOptions.KERBEROS_TICKET_LIFETIME) + .toMillis(); } } catch (Exception e) { throw new IllegalStateException("Failed to resolve kerberos TTL configuration", e); @@ -228,34 +210,34 @@ private static void closeHadoop() { private static UserGroupInformation getKerberosUGI() { LOG.info("[StreamPark] kerberos login starting...."); - if (HadoopConfigUtils.KERBEROS_PRINCIPAL.isEmpty() - || HadoopConfigUtils.KERBEROS_KEYTAB.isEmpty()) { + String principal = HadoopConfigUtils.kerberosPrincipal(); + String keytab = HadoopConfigUtils.kerberosKeytab(); + if (principal.isEmpty() || keytab.isEmpty()) { throw new IllegalArgumentException( - ConfigKeys.KEY_SECURITY_KERBEROS_PRINCIPAL() + HadoopOptions.KERBEROS_PRINCIPAL.key() + " and " - + ConfigKeys.KEY_SECURITY_KERBEROS_KEYTAB() + + HadoopOptions.KERBEROS_KEYTAB.key() + " must not be empty"); } System.setProperty("javax.security.auth.useSubjectCredsOnly", "false"); - if (!HadoopConfigUtils.KERBEROS_KRB5.isEmpty()) { - System.setProperty("java.security.krb5.conf", HadoopConfigUtils.KERBEROS_KRB5); - System.setProperty("java.security.krb5.conf.path", HadoopConfigUtils.KERBEROS_KRB5); + String krb5 = HadoopConfigUtils.kerberosKrb5(); + if (!krb5.isEmpty()) { + System.setProperty("java.security.krb5.conf", krb5); + System.setProperty("java.security.krb5.conf.path", krb5); } - System.setProperty("sun.security.spnego.debug", HadoopConfigUtils.KERBEROS_DEBUG); - System.setProperty("sun.security.krb5.debug", HadoopConfigUtils.KERBEROS_DEBUG); - hadoopConf() - .set( - ConfigKeys.KEY_HADOOP_SECURITY_AUTHENTICATION(), - ConfigKeys.KEY_KERBEROS()); + String debug = Boolean.toString(HadoopConfigUtils.kerberosDebug()); + System.setProperty("sun.security.spnego.debug", debug); + System.setProperty("sun.security.krb5.debug", debug); + hadoopConf().set("hadoop.security.authentication", "kerberos"); try { UserGroupInformation.setConfiguration(hadoopConf()); UserGroupInformation kerberosUgi = UserGroupInformation.loginUserFromKeytabAndReturnUGI( - HadoopConfigUtils.KERBEROS_PRINCIPAL, HadoopConfigUtils.KERBEROS_KEYTAB); + principal, keytab); UserGroupInformation.setLoginUser(kerberosUgi); LOG.info("[StreamPark] kerberos authentication successful"); return kerberosUgi; @@ -267,17 +249,14 @@ private static UserGroupInformation getKerberosUGI() { public static FileSystem hdfs() { if (reusableHdfs == null) { try { - reusableHdfs = - getUgi() - .doAs( - (PrivilegedAction) () -> { - try { - return FileSystem.get(hadoopConf()); - } catch (IOException e) { - throw new IllegalStateException("Failed to obtain HDFS FileSystem", e); - } - }); - if (HadoopConfigUtils.KERBEROS_ENABLE) { + reusableHdfs = getUgi().doAs((PrivilegedAction) () -> { + try { + return FileSystem.get(hadoopConf()); + } catch (IOException e) { + throw new IllegalStateException("Failed to obtain HDFS FileSystem", e); + } + }); + if (HadoopConfigUtils.kerberosEnabled()) { Timer timer = new Timer(); long refreshTime = getTgtRefreshTime(); timer.schedule( @@ -304,17 +283,15 @@ public void run() { public static YarnClient yarnClient() { if (reusableYarnClient == null || !reusableYarnClient.isInState(Service.STATE.STARTED)) { try { - reusableYarnClient = - getUgi() - .doAs( - (PrivilegedAction) () -> { - YarnConfiguration yarnConf = - new YarnConfiguration(hadoopConf()); - YarnClient client = YarnClient.createYarnClient(); - client.init(yarnConf); - client.start(); - return client; - }); + reusableYarnClient = getUgi().doAs( + (PrivilegedAction) () -> { + YarnConfiguration yarnConf = + new YarnConfiguration(hadoopConf()); + YarnClient client = YarnClient.createYarnClient(); + client.init(yarnConf); + client.start(); + return client; + }); } catch (Exception e) { throw new IllegalArgumentException( "[StreamPark] access yarnClient error: " + e, e); diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java deleted file mode 100644 index 0af294fd8d..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/JdbcUtils.java +++ /dev/null @@ -1,332 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -import org.apache.streampark.common.conf.ConfigKeys; - -import org.apache.streampark.shaded.org.slf4j.Logger; - -import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; - -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantLock; -import java.util.function.Consumer; - -/** - * Based on the hikari connection pool implementation. Support multiple data sources, note that all - * modifications and additions are automatically committed transactions. - */ -public final class JdbcUtils { - - private static final Logger LOG = - StreamParkLoggerFactory.loggerFactory().getLogger(JdbcUtils.class.getName()); - - private static final ConcurrentHashMap LOCK_MAP = new ConcurrentHashMap<>(); - - private static final ConcurrentHashMap DATA_SOURCE_HOLDER = - new ConcurrentHashMap<>(); - - private JdbcUtils() { - } - - public static List> select(String sql, Properties jdbcConfig) { - return select(sql, null, jdbcConfig); - } - - public static List> select( - String sql, Consumer func, Properties jdbcConfig) { - if (sql == null || sql.isEmpty()) { - return Collections.emptyList(); - } - Connection conn = getConnection(jdbcConfig); - Statement stmt = null; - ResultSet result = null; - try { - stmt = createStatement(conn); - result = stmt.executeQuery(sql); - if (func != null) { - func.accept(result); - } - int count = result.getMetaData().getColumnCount(); - List> array = new ArrayList<>(); - while (result.next()) { - Map map = new HashMap<>(); - for (int x = 1; x <= count; x++) { - String key = result.getMetaData().getColumnLabel(x); - Object value = result.getObject(x); - map.put(key, value); - } - array.add(map); - } - return array.isEmpty() ? Collections.emptyList() : array; - } catch (Exception ex) { - LOG.warn("Failed to execute JDBC select", ex); - return Collections.emptyList(); - } finally { - close(result, stmt, conn); - } - } - - public static long count(String sql, Properties jdbcConfig) { - Map row = unique(sql, jdbcConfig); - if (row.isEmpty()) { - return 0L; - } - return Long.parseLong(row.values().iterator().next().toString()); - } - - public static long count(Connection conn, String sql) { - Map row = unique(conn, sql); - if (row.isEmpty()) { - return 0L; - } - return Long.parseLong(row.values().iterator().next().toString()); - } - - public static int batch(Iterable sql, Properties jdbcConfig) { - int size = 0; - for (String ignored : sql) { - size++; - } - if (size == 0) { - return 0; - } - if (size == 1) { - return update(sql.iterator().next(), jdbcConfig); - } - Connection conn = getConnection(jdbcConfig); - try { - Statement prepStat = conn.createStatement(); - try { - int index = 0; - int batchSize = 1000; - int total = 0; - for (String x : sql) { - prepStat.addBatch(x); - index++; - if (index > 0 && index % batchSize == 0) { - int count = 0; - for (int c : prepStat.executeBatch()) { - count += c; - } - conn.commit(); - prepStat.clearBatch(); - total += count; - } - } - for (int c : prepStat.executeBatch()) { - total += c; - } - return total; - } catch (Exception ex) { - LOG.warn("Failed to execute JDBC select", ex); - return 0; - } finally { - conn.commit(); - close(conn); - } - } catch (Exception ex) { - LOG.warn("Failed to execute JDBC select", ex); - return 0; - } - } - - public static int update(String sql, Properties jdbcConfig) { - return update(getConnection(jdbcConfig), sql); - } - - public static int update(Connection conn, String sql) { - Statement statement = null; - try { - statement = conn.createStatement(); - return statement.executeUpdate(sql); - } catch (Exception ex) { - LOG.warn("Failed to execute JDBC select", ex); - return -1; - } finally { - close(statement, conn); - } - } - - public static Map unique(String sql, Properties jdbcConfig) { - return unique(getConnection(jdbcConfig), sql); - } - - public static Map unique(Connection conn, String sql) { - Statement stmt = null; - ResultSet result = null; - try { - stmt = createStatement(conn); - result = stmt.executeQuery(sql); - int count = result.getMetaData().getColumnCount(); - if (!result.next()) { - return Collections.emptyMap(); - } - Map map = new HashMap<>(); - for (int x = 1; x <= count; x++) { - String key = result.getMetaData().getColumnLabel(x); - Object value = result.getObject(x); - map.put(key, value); - } - return map; - } catch (Exception ex) { - LOG.warn("Failed to execute JDBC select", ex); - return Collections.emptyMap(); - } finally { - close(result, stmt, conn); - } - } - - public static boolean execute(String sql, Properties jdbcConfig) { - return execute(getConnection(jdbcConfig), sql); - } - - public static boolean execute(Connection conn, String sql) { - Statement stmt = null; - try { - stmt = conn.createStatement(); - return stmt.execute(sql); - } catch (Exception ex) { - LOG.warn("Failed to execute JDBC select", ex); - return false; - } finally { - close(stmt, conn); - } - } - - public static Connection getConnection(Properties prop) { - String alias = prop.getProperty(ConfigKeys.KEY_ALIAS()); - ReentrantLock lock = LOCK_MAP.computeIfAbsent(alias, k -> new ReentrantLock()); - lock.lock(); - try { - HikariDataSource ds = DATA_SOURCE_HOLDER.get(alias); - if (ds == null) { - HikariConfig jdbcConfig = new HikariConfig(); - for (String key : prop.stringPropertyNames()) { - if (ConfigKeys.KEY_ALIAS().equals(key) || ConfigKeys.KEY_SEMANTIC().equals(key)) { - continue; - } - String value = prop.getProperty(key); - try { - Field field = jdbcConfig.getClass().getDeclaredField(key); - field.setAccessible(true); - setFieldValue(field, jdbcConfig, value); - } catch (NoSuchFieldException e) { - String setMethod = - "set" + key.substring(0, 1).toUpperCase() + key.substring(1); - Method method = null; - for (Method m : jdbcConfig.getClass().getMethods()) { - if (m.getName().equals(setMethod) && m.getParameterCount() == 1) { - method = m; - break; - } - } - if (method != null) { - method.setAccessible(true); - invokeSetter(method, jdbcConfig, value); - } else { - throw new IllegalArgumentException( - "jdbcConfig error,property:" - + key - + " invalid,please see more properties jdbcConfig https://github.com/brettwooldridge/HikariCP"); - } - } - } - ds = new HikariDataSource(jdbcConfig); - DATA_SOURCE_HOLDER.put(alias, ds); - } - return ds.getConnection(); - } catch (Exception e) { - throw new IllegalStateException("Failed to obtain JDBC connection for alias: " + alias, e); - } finally { - lock.unlock(); - } - } - - private static void setFieldValue(Field field, HikariConfig jdbcConfig, - String value) throws IllegalAccessException { - String type = field.getType().getSimpleName(); - switch (type) { - case "String": - field.set(jdbcConfig, value); - break; - case "int": - field.setInt(jdbcConfig, Integer.parseInt(value)); - break; - case "long": - field.setLong(jdbcConfig, Long.parseLong(value)); - break; - case "boolean": - field.setBoolean(jdbcConfig, Boolean.parseBoolean(value)); - break; - default: - break; - } - } - - private static void invokeSetter(Method method, HikariConfig jdbcConfig, String value) throws Exception { - String type = method.getParameterTypes()[0].getSimpleName(); - switch (type) { - case "String": - method.invoke(jdbcConfig, value); - break; - case "int": - case "Integer": - method.invoke(jdbcConfig, Integer.parseInt(value)); - break; - case "long": - case "Long": - method.invoke(jdbcConfig, Long.parseLong(value)); - break; - case "boolean": - case "Boolean": - method.invoke(jdbcConfig, Boolean.parseBoolean(value)); - break; - default: - break; - } - } - - private static Statement createStatement(Connection conn) throws Exception { - return conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY); - } - - public static void close(AutoCloseable... closeable) { - for (AutoCloseable item : closeable) { - if (item != null) { - try { - item.close(); - } catch (Exception ignored) { - // ignore - } - } - } - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/MongoConfig.java b/streampark-common/src/main/java/org/apache/streampark/common/util/MongoConfig.java deleted file mode 100644 index 374815cd9c..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/MongoConfig.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -import org.apache.streampark.common.conf.ConfigKeys; - -import com.mongodb.MongoClient; -import com.mongodb.MongoClientOptions; -import com.mongodb.MongoClientURI; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; - -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; -import java.util.regex.Pattern; - -/** MongoDB client configuration helper. */ -public final class MongoConfig { - - private static final Pattern LEADING_DOT_PATTERN = Pattern.compile("^\\."); - - public static final String CLIENT_URI = "client-uri"; - public static final String ADDRESS = "address"; - public static final String REPLICA_SET = "replica-set"; - public static final String DATABASE = "database"; - public static final String USERNAME = "username"; - public static final String PASSWORD = "password"; - public static final String MIN_CONNECTIONS_PER_HOST = "min-connections-per-host"; - public static final String MAX_CONNECTIONS_PER_HOST = "max-connections-per-host"; - public static final String THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER = - "threads-allowed-to-block-for-connection-multiplier"; - public static final String SERVER_SELECTION_TIMEOUT = "server-selection-timeout"; - public static final String MAX_WAIT_TIME = "max-wait-time"; - public static final String MAX_CONNECTION_IDEL_TIME = "max-connection-idel-time"; - public static final String MAX_CONNECTION_LIFE_TIME = "max-connection-life-time"; - public static final String CONNECT_TIMEOUT = "connect-timeout"; - public static final String SOCKET_TIMEOUT = "socket-timeout"; - public static final String SOCKET_KEEP_ALIVE = "socket-keep-alive"; - public static final String SSL_ENABLED = "ssl-enabled"; - public static final String SSL_INVALID_HOST_NAME_ALLOWED = "ssl-invalid-host-name-allowed"; - public static final String ALWAYS_USE_M_BEANS = "always-use-m-beans"; - public static final String HEARTBEAT_SOCKET_TIMEOUT = "heartbeat-socket-timeout"; - public static final String HEARTBEAT_CONNECT_TIMEOUT = "heartbeat-connect-timeout"; - public static final String MIN_HEARTBEAT_FREQUENCY = "min-heartbeat-frequency"; - public static final String HEARTBEAT_FREQUENCY = "heartbeat-frequency"; - public static final String LOCAL_THRESHOLD = "local-threshold"; - public static final String AUTHENTICATION_DATABASE = "authentication-database"; - - private MongoConfig() { - } - - public static String getProperty(Properties properties, String k) { - return getProperty(properties, k, ""); - } - - public static String getProperty(Properties properties, String k, String alias) { - return getProperties(properties, alias).getProperty(k); - } - - public static Properties getProperties(Properties properties) { - return getProperties(properties, ""); - } - - public static Properties getProperties(Properties properties, String alias) { - Properties prop = new Properties(); - for (String key : properties.stringPropertyNames()) { - if (!key.startsWith(ConfigKeys.MONGO_PREFIX())) { - continue; - } - String value = properties.getProperty(key); - if (value == null || value.isEmpty()) { - continue; - } - String stripped = key.replace(ConfigKeys.MONGO_PREFIX() + alias, ""); - String k = LEADING_DOT_PATTERN.matcher(stripped).replaceFirst(""); - prop.put(k, value.trim()); - } - return prop; - } - - public static MongoClient getClient(Properties properties) { - return getClient(properties, ""); - } - - public static MongoClient getClient(Properties properties, String alias) { - Properties mongoParam = getProperties(properties, alias); - if (mongoParam.containsKey(CLIENT_URI)) { - MongoClientURI clientURI = new MongoClientURI(mongoParam.getProperty(CLIENT_URI)); - return new MongoClient(clientURI); - } - MongoClientOptions.Builder builder = new MongoClientOptions.Builder(); - if (mongoParam.containsKey(MAX_CONNECTIONS_PER_HOST)) { - builder.connectionsPerHost(Integer.parseInt(mongoParam.getProperty(MAX_CONNECTIONS_PER_HOST))); - } - if (mongoParam.containsKey(MIN_CONNECTIONS_PER_HOST)) { - builder.minConnectionsPerHost(Integer.parseInt(mongoParam.getProperty(MIN_CONNECTIONS_PER_HOST))); - } - if (mongoParam.containsKey(REPLICA_SET)) { - builder.requiredReplicaSetName(mongoParam.getProperty(REPLICA_SET)); - } - if (mongoParam.containsKey(THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER)) { - builder.threadsAllowedToBlockForConnectionMultiplier( - Integer.parseInt(mongoParam.getProperty(THREADS_ALLOWED_TO_BLOCK_FOR_CONNECTION_MULTIPLIER))); - } - if (mongoParam.containsKey(SERVER_SELECTION_TIMEOUT)) { - builder.serverSelectionTimeout(Integer.parseInt(mongoParam.getProperty(SERVER_SELECTION_TIMEOUT))); - } - if (mongoParam.containsKey(MAX_WAIT_TIME)) { - builder.maxWaitTime(Integer.parseInt(mongoParam.getProperty(MAX_WAIT_TIME))); - } - if (mongoParam.containsKey(MAX_CONNECTION_IDEL_TIME)) { - builder.maxConnectionIdleTime(Integer.parseInt(mongoParam.getProperty(MAX_CONNECTION_IDEL_TIME))); - } - if (mongoParam.containsKey(MAX_CONNECTION_LIFE_TIME)) { - builder.maxConnectionLifeTime(Integer.parseInt(mongoParam.getProperty(MAX_CONNECTION_LIFE_TIME))); - } - if (mongoParam.containsKey(CONNECT_TIMEOUT)) { - builder.connectTimeout(Integer.parseInt(mongoParam.getProperty(CONNECT_TIMEOUT))); - } - if (mongoParam.containsKey(SOCKET_TIMEOUT)) { - builder.socketTimeout(Integer.parseInt(mongoParam.getProperty(SOCKET_TIMEOUT))); - } - if (mongoParam.containsKey(SSL_ENABLED)) { - builder.sslEnabled(Boolean.parseBoolean(mongoParam.getProperty(SSL_ENABLED))); - } - if (mongoParam.containsKey(SSL_INVALID_HOST_NAME_ALLOWED)) { - builder.sslInvalidHostNameAllowed( - Boolean.parseBoolean(mongoParam.getProperty(SSL_INVALID_HOST_NAME_ALLOWED))); - } - if (mongoParam.containsKey(ALWAYS_USE_M_BEANS)) { - builder.alwaysUseMBeans(Boolean.parseBoolean(mongoParam.getProperty(ALWAYS_USE_M_BEANS))); - } - if (mongoParam.containsKey(HEARTBEAT_FREQUENCY)) { - builder.heartbeatFrequency(Integer.parseInt(mongoParam.getProperty(HEARTBEAT_FREQUENCY))); - } - if (mongoParam.containsKey(MIN_HEARTBEAT_FREQUENCY)) { - builder.minHeartbeatFrequency(Integer.parseInt(mongoParam.getProperty(MIN_HEARTBEAT_FREQUENCY))); - } - if (mongoParam.containsKey(HEARTBEAT_CONNECT_TIMEOUT)) { - builder.heartbeatConnectTimeout(Integer.parseInt(mongoParam.getProperty(HEARTBEAT_CONNECT_TIMEOUT))); - } - if (mongoParam.containsKey(HEARTBEAT_SOCKET_TIMEOUT)) { - builder.heartbeatSocketTimeout(Integer.parseInt(mongoParam.getProperty(HEARTBEAT_SOCKET_TIMEOUT))); - } - if (mongoParam.containsKey(LOCAL_THRESHOLD)) { - builder.localThreshold(Integer.parseInt(mongoParam.getProperty(LOCAL_THRESHOLD))); - } - MongoClientOptions mongoClientOptions = builder.build(); - String[] addresses = mongoParam.getProperty(ADDRESS).split(","); - List serverAddresses = new ArrayList<>(); - for (String x : addresses) { - String[] hostAndPort = x.split(":"); - String host = hostAndPort[0]; - int port = Integer.parseInt(hostAndPort[1]); - serverAddresses.add(new ServerAddress(host, port)); - } - if (mongoParam.containsKey(USERNAME)) { - String db; - if (mongoParam.containsKey(AUTHENTICATION_DATABASE)) { - db = mongoParam.getProperty(AUTHENTICATION_DATABASE); - } else { - db = mongoParam.getProperty(DATABASE); - } - MongoCredential mongoCredential = - MongoCredential.createScramSha1Credential( - mongoParam.getProperty(USERNAME), - db, - mongoParam.getProperty(PASSWORD).toCharArray()); - return new MongoClient(serverAddresses, mongoCredential, mongoClientOptions); - } - return new MongoClient(serverAddresses, mongoClientOptions); - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/OkHttpUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/OkHttpUtils.java new file mode 100644 index 0000000000..0329d0580a --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/OkHttpUtils.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import okhttp3.ConnectionPool; +import okhttp3.Dispatcher; +import okhttp3.Interceptor; +import okhttp3.OkHttpClient; +import okhttp3.Protocol; +import okhttp3.Request; +import okhttp3.Response; + +import java.io.IOException; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +/** Shared OkHttp client for StreamPark's outbound HTTP requests. */ +public final class OkHttpUtils { + + private static final long CONNECT_TIMEOUT_SECONDS = 30L; + private static final long READ_TIMEOUT_SECONDS = 60L; + private static final long WRITE_TIMEOUT_SECONDS = 60L; + private static final long CALL_TIMEOUT_SECONDS = 120L; + private static final int MAX_RETRY_ATTEMPTS = 3; + + private static final Interceptor RETRY_INTERCEPTOR = newRetryInterceptor(); + private static final OkHttpClient HTTP_CLIENT = createClient(); + + private OkHttpUtils() { + } + + /** + * Executes an HTTP request with the shared connection pool and retry policy. + * + *

The caller owns the returned response and must close it. + * + * @param request outbound request + * @return open upstream response + * @throws IOException when the request cannot be completed + */ + public static Response call(Request request) throws IOException { + return execute(request, HTTP_CLIENT); + } + + private static Response execute(Request request, OkHttpClient client) throws IOException { + try { + return client.newCall(request).execute(); + } catch (IOException e) { + throw new IOException( + "Failed to execute HTTP request: " + request.method() + " " + request.url().encodedPath(), + e); + } + } + + private static OkHttpClient createClient() { + Dispatcher dispatcher = new Dispatcher(); + dispatcher.setMaxRequests(256); + dispatcher.setMaxRequestsPerHost(32); + return new OkHttpClient.Builder() + .connectionPool(new ConnectionPool(32, 5L, TimeUnit.MINUTES)) + .connectTimeout(CONNECT_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .readTimeout(READ_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .writeTimeout(WRITE_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .callTimeout(CALL_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .retryOnConnectionFailure(true) + .addInterceptor(RETRY_INTERCEPTOR) + .protocols(Arrays.asList(Protocol.HTTP_2, Protocol.HTTP_1_1)) + .dispatcher(dispatcher) + .build(); + } + + /** Retries transient failures only for methods whose requests are safe to replay. */ + private static Interceptor newRetryInterceptor() { + return chain -> { + Request request = chain.request(); + if (!isRetryableMethod(request.method())) { + return chain.proceed(request); + } + + Response response = null; + for (int attempt = 1; attempt <= MAX_RETRY_ATTEMPTS; attempt++) { + try { + response = chain.proceed(request); + if (response.isSuccessful() + || !isRetryableStatus(response.code()) + || attempt == MAX_RETRY_ATTEMPTS) { + return response; + } + response.close(); + response = null; + } catch (IOException e) { + if (attempt == MAX_RETRY_ATTEMPTS) { + throw e; + } + } + + try { + Thread.sleep((1L << attempt) * 1000L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while retrying HTTP request", e); + } + } + throw new IOException("HTTP retry loop ended without a response"); + }; + } + + private static boolean isRetryableMethod(String method) { + return "GET".equals(method) || "HEAD".equals(method) || "OPTIONS".equals(method); + } + + private static boolean isRetryableStatus(int statusCode) { + return statusCode == 408 || statusCode == 429 || (statusCode >= 500 && statusCode < 600); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/SafePathUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/PathUtils.java similarity index 78% rename from streampark-common/src/main/java/org/apache/streampark/common/util/SafePathUtils.java rename to streampark-common/src/main/java/org/apache/streampark/common/util/PathUtils.java index 9e76095863..e9da9466fc 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/SafePathUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/PathUtils.java @@ -25,13 +25,13 @@ import java.nio.file.Paths; /** Path resolution helpers for user-supplied file locations. */ -public final class SafePathUtils { +public final class PathUtils { - private SafePathUtils() { + private PathUtils() { } - public static Path resolveConfigPath(String filename) { - validateConfigFilename(filename); + public static Path resolvePath(String filename) { + validateFilename(filename); Path base = Paths.get("").toAbsolutePath().normalize(); Path resolved = base.resolve(filename).normalize(); if (!Paths.get(filename).isAbsolute() && !resolved.startsWith(base)) { @@ -40,8 +40,8 @@ public static Path resolveConfigPath(String filename) { return resolved; } - public static InputStream openConfigFile(String filename) throws IOException { - validateConfigFilename(filename); + public static InputStream openFile(String filename) throws IOException { + validateFilename(filename); Path base = Paths.get("").toAbsolutePath().normalize(); Path resolved = base.resolve(filename).normalize(); if (!Paths.get(filename).isAbsolute() && !resolved.startsWith(base)) { @@ -50,15 +50,11 @@ public static InputStream openConfigFile(String filename) throws IOException { return Files.newInputStream(resolved); // NOSONAR javasecurity:S2083 - path validated via Path.resolve above } - public static String readConfigFile(String filename) throws IOException { - validateConfigFilename(filename); - Path base = Paths.get("").toAbsolutePath().normalize(); - Path resolved = base.resolve(filename).normalize(); - if (!Paths.get(filename).isAbsolute() && !resolved.startsWith(base)) { - throw new IOException("invalid file path: " + filename); - } - return Files.readString(resolved, StandardCharsets.UTF_8); // NOSONAR javasecurity:S2083 - path validated via - // Path.resolve above + public static String readFileAsString(String filename) throws IOException { + Path resolved = resolvePath(filename); + // NOSONAR javasecurity:S2083 - path validated via + // Path.resolve above + return Files.readString(resolved, StandardCharsets.UTF_8); } public static Path resolveJarPath(java.net.URL jar) throws IOException { @@ -88,7 +84,7 @@ public static InputStream openJarFile(java.net.URL jar) throws IOException { } } - private static void validateConfigFilename(String filename) { + private static void validateFilename(String filename) { if (filename == null || filename.isEmpty()) { throw new IllegalArgumentException("filename must not be blank"); } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/PropertiesUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/PropertiesUtils.java deleted file mode 100644 index 01410bd3c6..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/PropertiesUtils.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -import org.apache.commons.lang3.StringUtils; - -import com.typesafe.config.ConfigFactory; -import org.yaml.snakeyaml.Yaml; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.io.StringReader; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collection; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.Map; -import java.util.Properties; -import java.util.Scanner; -import java.util.regex.Pattern; - -/** Properties, YAML and HOCON configuration parsing utilities. */ -public final class PropertiesUtils { - - private static final Pattern HOCON_KEY_QUOTE_PATTERN = Pattern.compile("\""); - - private PropertiesUtils() { - } - - public static String readFile(String filename) throws IOException { - Path path = SafePathUtils.resolveConfigPath(filename); - if (!Files.exists(path)) { - throw new IllegalArgumentException("[StreamPark] readFile: file " + path + " does not exist"); - } - if (!Files.isRegularFile(path)) { - throw new IllegalArgumentException("[StreamPark] readFile: file " + path + " is not a normal file"); - } - StringBuilder buffer = new StringBuilder(); - try (Scanner scanner = new Scanner(path)) { - while (scanner.hasNextLine()) { - buffer.append(scanner.nextLine()).append("\r\n"); - } - } - return buffer.toString(); - } - - public static LinkedHashMap fromYamlText(String text) { - try { - Map map = new Yaml().load(text); - return flatten(map); - } catch (RuntimeException e) { - throw new IllegalArgumentException("Failed when loading conf error:", e); - } - } - - public static LinkedHashMap fromHoconText(String conf) { - if (conf == null) { - throw new IllegalArgumentException("[StreamPark] fromHoconText: Hocon content must not be null"); - } - return parseHoconByReader(new StringReader(conf)); - } - - public static LinkedHashMap fromPropertiesText(String conf) { - try { - Properties properties = new Properties(); - properties.load(new StringReader(conf)); - LinkedHashMap result = new LinkedHashMap<>(); - for (String key : properties.stringPropertyNames()) { - result.put(key, properties.getProperty(key).trim()); - } - return result; - } catch (IOException e) { - throw new IllegalArgumentException("Failed when loading properties ", e); - } - } - - /** Load Yaml present in the given file. */ - public static LinkedHashMap fromYamlFile(String filename) { - try (InputStream inputStream = SafePathUtils.openConfigFile(filename)) { - return fromYamlFile(inputStream); - } catch (IOException e) { - throw new IllegalArgumentException("Failed when loading yaml from file " + filename, e); - } - } - - public static LinkedHashMap fromHoconFile(String filename) { - try (InputStream inputStream = SafePathUtils.openConfigFile(filename)) { - return fromHoconFile(inputStream); - } catch (IOException e) { - throw new IllegalArgumentException("Failed when loading Hocon from file " + filename, e); - } - } - - /** Load properties present in the given file. */ - public static LinkedHashMap fromPropertiesFile(String filename) { - try (InputStream inputStream = SafePathUtils.openConfigFile(filename)) { - return fromPropertiesFile(inputStream); - } catch (IOException e) { - throw new IllegalArgumentException( - "[StreamPark] Failed when loading properties from file " + filename, e); - } - } - - /** Load Yaml present in the given input stream. */ - public static LinkedHashMap fromYamlFile(InputStream inputStream) { - AssertUtils.required( - inputStream != null, - "[StreamPark] fromYamlFile: Properties inputStream must not be null"); - try { - Map map = new Yaml().load(inputStream); - return flatten(map); - } catch (RuntimeException e) { - throw new IllegalArgumentException("Failed when loading yaml from inputStream", e); - } - } - - public static LinkedHashMap fromHoconFile(InputStream inputStream) { - if (inputStream == null) { - throw new IllegalArgumentException("[StreamPark] fromHoconFile: Hocon inputStream must not be null"); - } - return parseHoconByReader(new InputStreamReader(inputStream)); - } - - private static LinkedHashMap parseHoconByReader(Reader reader) { - try { - LinkedHashMap result = new LinkedHashMap<>(); - ConfigFactory.parseReader(reader) - .entrySet() - .forEach( - entry -> { - String key = HOCON_KEY_QUOTE_PATTERN.matcher(entry.getKey().trim()).replaceAll(""); - String value = entry.getValue().unwrapped().toString().trim(); - result.put(key, value); - }); - return result; - } catch (RuntimeException e) { - throw new IllegalArgumentException("Failed when loading Hocon ", e); - } - } - - /** Load properties present in the given input stream. */ - public static LinkedHashMap fromPropertiesFile(InputStream inputStream) { - if (inputStream == null) { - throw new IllegalArgumentException( - "[StreamPark] fromPropertiesFile: Properties inputStream must not be null"); - } - try { - Properties properties = new Properties(); - properties.load(inputStream); - LinkedHashMap result = new LinkedHashMap<>(); - for (String key : properties.stringPropertyNames()) { - result.put(key, properties.getProperty(key).trim()); - } - return result; - } catch (IOException e) { - throw new IllegalArgumentException( - "[StreamPark] Failed when loading properties from inputStream", e); - } - } - - public static Map fromYamlTextAsJava(String text) { - return new HashMap<>(fromYamlText(text)); - } - - public static Map fromHoconTextAsJava(String text) { - return new HashMap<>(fromHoconText(text)); - } - - public static Map fromPropertiesTextAsJava(String text) { - return new HashMap<>(fromPropertiesText(text)); - } - - public static Map fromYamlFileAsJava(String filename) { - return new HashMap<>(fromYamlFile(filename)); - } - - public static Map fromHoconFileAsJava(String filename) { - return new HashMap<>(fromHoconFile(filename)); - } - - public static Map fromPropertiesFileAsJava(String filename) { - return new HashMap<>(fromPropertiesFile(filename)); - } - - public static Map fromYamlFileAsJava(InputStream inputStream) { - return new HashMap<>(fromYamlFile(inputStream)); - } - - public static Map fromHoconFileAsJava(InputStream inputStream) { - return new HashMap<>(fromHoconFile(inputStream)); - } - - public static Map fromPropertiesFileAsJava(InputStream inputStream) { - return new HashMap<>(fromPropertiesFile(inputStream)); - } - - private static LinkedHashMap flatten(Map map) { - return flatten(map, ""); - } - - private static LinkedHashMap flatten(Map map, String prefix) { - if (map == null || map.isEmpty()) { - return new LinkedHashMap<>(); - } - LinkedHashMap result = new LinkedHashMap<>(); - for (Map.Entry entry : map.entrySet()) { - String key = entry.getKey(); - Object value = entry.getValue(); - if (value instanceof Map) { - @SuppressWarnings("unchecked") - Map nested = (Map) value; - result.putAll(flatten(nested, prefix + key + ".")); - } else if (value instanceof String) { - if (StringUtils.isNotBlank((String) value)) { - result.put(prefix + key, (String) value); - } - } else if (value instanceof Collection) { - Collection collection = (Collection) value; - if (!collection.isEmpty()) { - result.put(prefix + key, collection.toString()); - } - } else if (value != null) { - result.put(prefix + key, value.toString()); - } - } - return result; - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/RedisClient.java b/streampark-common/src/main/java/org/apache/streampark/common/util/RedisClient.java deleted file mode 100644 index 00b60b4047..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/RedisClient.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -import org.apache.streampark.common.constants.Constants; - -import org.apache.streampark.shaded.org.slf4j.Logger; - -import redis.clients.jedis.HostAndPort; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.JedisPool; -import redis.clients.jedis.JedisPoolConfig; -import redis.clients.jedis.exceptions.JedisConnectionException; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadLocalRandom; - -/** Redis connection pool and cluster client factory. */ -public final class RedisClient { - - private static final Logger LOG = - StreamParkLoggerFactory.loggerFactory().getLogger(RedisClient.class.getName()); - - private static final ConcurrentHashMap POOLS = new ConcurrentHashMap<>(); - - private static final ConcurrentHashMap CLUSTERS = new ConcurrentHashMap<>(); - - private static final JedisPoolConfig POOL_CONFIG; - - static { - JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxTotal(1000); - poolConfig.setMaxIdle(64); - poolConfig.setTestOnBorrow(true); - poolConfig.setTestOnReturn(false); - poolConfig.setTestWhileIdle(false); - poolConfig.setMinEvictableIdleTimeMillis(1800000); - poolConfig.setTimeBetweenEvictionRunsMillis(30000); - poolConfig.setNumTestsPerEvictionRun(-1); - POOL_CONFIG = poolConfig; - } - - private RedisClient() { - } - - public static Jedis connect(RedisEndpoint[] endpoints) { - if (endpoints.length == 0) { - throw new IllegalArgumentException("[StreamPark] The RedisEndpoint array is empty!!!"); - } - int index = ThreadLocalRandom.current().nextInt(endpoints.length); - try { - return connect(endpoints[index]); - } catch (Exception e) { - LOG.error("[StreamPark] {}", e.getMessage()); - RedisEndpoint[] remaining = new RedisEndpoint[endpoints.length - index - 1]; - System.arraycopy(endpoints, index + 1, remaining, 0, remaining.length); - if (remaining.length == 0) { - throw e; - } - return connect(remaining); - } - } - - public static Jedis connect(RedisEndpoint endpoint) { - JedisPool pool = POOLS.computeIfAbsent(endpoint, RedisClient::createJedisPool); - int sleepTime = 4; - Jedis conn = null; - while (conn == null) { - try { - conn = pool.getResource(); - } catch (JedisConnectionException e) { - if (e.getCause() != null - && e.getCause().toString().contains("ERR max number of clients reached")) { - if (sleepTime < 500) { - sleepTime *= 2; - } - try { - Thread.sleep(sleepTime); - } catch (InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Interrupted while waiting for Redis connection", ie); - } - } else { - throw e; - } - } - } - return conn; - } - - public static JedisPool createJedisPool(RedisEndpoint endpoint) { - RedisEndpoint endpointEn = endpoint.withAuth(Constants.DEFAULT_DATAMASK_STRING); - LOG.info("[StreamPark] RedisClient: createJedisPool with {}", endpointEn); - return new JedisPool( - POOL_CONFIG, - endpoint.getHost(), - endpoint.getPort(), - endpoint.getTimeout(), - endpoint.getAuth(), - endpoint.getDb()); - } - - public static JedisCluster connectCluster(RedisEndpoint... endpoints) { - if (endpoints.length == 0) { - throw new IllegalArgumentException("[StreamPark] The RedisEndpoint array is empty!!!"); - } - RedisEndpoint head = endpoints[0]; - return CLUSTERS.computeIfAbsent( - head, - key -> { - Set hostPorts = new HashSet<>(); - for (RedisEndpoint r : endpoints) { - hostPorts.add(new HostAndPort(r.getHost(), r.getPort())); - } - return new JedisCluster(hostPorts, head.getTimeout(), 1000, 1, head.getAuth(), POOL_CONFIG); - }); - } - - public static void close() { - for (Map.Entry entry : POOLS.entrySet()) { - entry.getValue().close(); - } - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/RedisEndpoint.java b/streampark-common/src/main/java/org/apache/streampark/common/util/RedisEndpoint.java deleted file mode 100644 index 8bbcdddd63..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/RedisEndpoint.java +++ /dev/null @@ -1,377 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -import org.apache.streampark.common.conf.ConfigKeys; - -import redis.clients.jedis.Jedis; -import redis.clients.jedis.Protocol; -import redis.clients.jedis.util.JedisClusterCRC16; -import redis.clients.jedis.util.JedisURIHelper; -import redis.clients.jedis.util.SafeEncoder; - -import java.io.Serializable; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.ThreadLocalRandom; - -/** - * RedisEndpoint represents a redis connection endpoint info: host, port, auth password db number, - * and timeout. - */ -public class RedisEndpoint implements Serializable { - - private static final long serialVersionUID = 1L; - - private final String host; - private final int port; - private final String auth; - private final int db; - private final int timeout; - - public RedisEndpoint() { - this(Protocol.DEFAULT_HOST, Protocol.DEFAULT_PORT, null, Protocol.DEFAULT_DATABASE, Protocol.DEFAULT_TIMEOUT); - } - - public RedisEndpoint(String host, int port, String auth, int db, int timeout) { - this.host = host; - this.port = port; - this.auth = auth; - this.db = db; - this.timeout = timeout; - } - - public RedisEndpoint(Properties conf) { - this( - conf.getProperty(ConfigKeys.KEY_HOST(), Protocol.DEFAULT_HOST), - Integer.parseInt( - conf.getProperty(ConfigKeys.KEY_PORT(), String.valueOf(Protocol.DEFAULT_PORT))), - conf.getProperty(ConfigKeys.KEY_PASSWORD(), null), - Integer.parseInt( - conf.getProperty(ConfigKeys.KEY_DB(), String.valueOf(Protocol.DEFAULT_DATABASE))), - Integer.parseInt( - conf.getProperty(ConfigKeys.KEY_TIMEOUT(), String.valueOf(Protocol.DEFAULT_TIMEOUT)))); - } - - public RedisEndpoint(URI uri) { - this( - uri.getHost(), - uri.getPort(), - JedisURIHelper.getPassword(uri), - JedisURIHelper.getDBIndex(uri), - Protocol.DEFAULT_TIMEOUT); - } - - public RedisEndpoint(String uri) { - this(URI.create(uri)); - } - - public String getHost() { - return host; - } - - public int getPort() { - return port; - } - - public String getAuth() { - return auth; - } - - public int getDb() { - return db; - } - - public int getTimeout() { - return timeout; - } - - public RedisEndpoint withAuth(String newAuth) { - return new RedisEndpoint(host, port, newAuth, db, timeout); - } - - public Jedis connect() { - return RedisClient.connect(this); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - RedisEndpoint that = (RedisEndpoint) o; - return port == that.port - && db == that.db - && timeout == that.timeout - && Objects.equals(host, that.host) - && Objects.equals(auth, that.auth); - } - - @Override - public int hashCode() { - return Objects.hash(host, port, auth, db, timeout); - } - - @Override - public String toString() { - return "RedisEndpoint{host='" - + host - + "', port=" - + port - + ", db=" - + db - + ", timeout=" - + timeout - + '}'; - } -} - -class RedisNode implements Serializable { - - private static final long serialVersionUID = 1L; - - private final RedisEndpoint endpoint; - private final int startSlot; - private final int endSlot; - private final int idx; - private final int total; - - RedisNode(RedisEndpoint endpoint, int startSlot, int endSlot, int idx, int total) { - this.endpoint = endpoint; - this.startSlot = startSlot; - this.endSlot = endSlot; - this.idx = idx; - this.total = total; - } - - public RedisEndpoint getEndpoint() { - return endpoint; - } - - public int getStartSlot() { - return startSlot; - } - - public int getEndSlot() { - return endSlot; - } - - public int getIdx() { - return idx; - } - - public int getTotal() { - return total; - } - - public Jedis connect() { - return endpoint.connect(); - } -} - -/** RedisConfig holds the state of the cluster nodes, and uses consistent hashing to map keys to nodes. */ -class RedisConfig implements Serializable { - - private static final long serialVersionUID = 1L; - - private final RedisEndpoint initialHost; - private final String initialAddr; - private final RedisNode[] hosts; - private final RedisNode[] nodes; - - RedisConfig(RedisEndpoint initialHost) { - this.initialHost = initialHost; - this.initialAddr = initialHost.getHost(); - this.nodes = getNodes(initialHost); - this.hosts = - Arrays.stream(nodes).filter(node -> node.getIdx() == 0).toArray(RedisNode[]::new); - } - - public String getInitialAddr() { - return initialAddr; - } - - public String getAuth() { - return initialHost.getAuth(); - } - - public int getDB() { - return initialHost.getDb(); - } - - public RedisNode getRandomNode() { - return hosts[ThreadLocalRandom.current().nextInt(hosts.length)]; - } - - public RedisNode[] getNodesBySlots(int sPos, int ePos) { - return Arrays.stream(nodes) - .filter(node -> intersect(sPos, ePos, node.getStartSlot(), node.getEndSlot())) - .filter(node -> node.getIdx() == 0) - .toArray(RedisNode[]::new); - } - - public Jedis connectionForKey(String key) { - return getHost(key).connect(); - } - - public RedisNode getHost(String key) { - int slot = JedisClusterCRC16.getSlot(key); - return Arrays.stream(hosts) - .filter(host -> host.getStartSlot() <= slot && host.getEndSlot() >= slot) - .findFirst() - .orElseThrow(() -> new IllegalStateException("[StreamPark] No redis host for key: " + key)); - } - - public RedisNode[] getNodes(RedisEndpoint endpoint) { - if (clusterEnabled(endpoint)) { - return getClusterNodes(endpoint); - } - return getNonClusterNodes(endpoint); - } - - private static boolean intersect(int sPos1, int ePos1, int sPos2, int ePos2) { - if (sPos1 <= sPos2) { - return ePos1 >= sPos2; - } - return ePos2 >= sPos1; - } - - private boolean clusterEnabled(RedisEndpoint endpoint) { - Jedis conn = endpoint.connect(); - try { - String[] info = conn.info().split("\n"); - String version = - Arrays.stream(info).filter(line -> line.contains("redis_version:")).findFirst().orElse(""); - String[] clusterEnable = - Arrays.stream(info).filter(line -> line.contains("cluster_enabled:")).toArray(String[]::new); - int mainVersion = Integer.parseInt(version.substring(14, version.indexOf('.'))); - return mainVersion > 2 && clusterEnable.length > 0 && clusterEnable[0].contains("1"); - } finally { - conn.close(); - } - } - - private RedisNode[] getNonClusterNodes(RedisEndpoint endpoint) { - String masterHost = endpoint.getHost(); - int masterPort = endpoint.getPort(); - Jedis conn = endpoint.connect(); - String[] replinfo; - try { - replinfo = conn.info("Replication").split("\n"); - } finally { - conn.close(); - } - - if (Arrays.stream(replinfo).anyMatch(line -> line.contains("role:slave"))) { - String host = - Arrays.stream(replinfo) - .filter(line -> line.contains("master_host:")) - .findFirst() - .orElse("") - .trim() - .substring(12); - int port = - Integer.parseInt( - Arrays.stream(replinfo) - .filter(line -> line.contains("master_port:")) - .findFirst() - .orElse("") - .trim() - .substring(12)); - return getNonClusterNodes( - new RedisEndpoint(host, port, endpoint.getAuth(), endpoint.getDb(), endpoint.getTimeout())); - } - - List slaves = new ArrayList<>(); - for (String rl : replinfo) { - if (rl.contains("slave") && rl.contains("online")) { - String content = rl.substring(rl.indexOf(':') + 1); - String[] parts = content.split(","); - String ipPart = parts[0]; - String portPart = parts[1]; - slaves.add( - new String[]{ - ipPart.substring(ipPart.indexOf('=') + 1), - portPart.substring(portPart.indexOf('=') + 1) - }); - } - } - - List nodeList = new ArrayList<>(); - nodeList.add(new RedisNode(endpoint, 0, 16383, 0, slaves.size() + 1)); - for (int i = 0; i < slaves.size(); i++) { - String[] slave = slaves.get(i); - nodeList.add( - new RedisNode( - new RedisEndpoint( - slave[0], - Integer.parseInt(slave[1]), - endpoint.getAuth(), - endpoint.getDb(), - endpoint.getTimeout()), - 0, - 16383, - i + 1, - slaves.size() + 1)); - } - return nodeList.toArray(new RedisNode[0]); - } - - private RedisNode[] getClusterNodes(RedisEndpoint endpoint) { - Jedis conn = endpoint.connect(); - try { - List slots = conn.clusterSlots(); - List result = new ArrayList<>(); - for (Object slotInfoObj : slots) { - @SuppressWarnings("unchecked") - List slotInfo = (List) slotInfoObj; - int sPos = Integer.parseInt(slotInfo.get(0).toString()); - int ePos = Integer.parseInt(slotInfo.get(1).toString()); - int total = slotInfo.size() - 2; - for (int i = 0; i < total; i++) { - @SuppressWarnings("unchecked") - List node = (List) slotInfo.get(i + 2); - String host = SafeEncoder.encode((byte[]) node.get(0)); - int port = Integer.parseInt(node.get(1).toString()); - result.add( - new RedisNode( - new RedisEndpoint( - host, - port, - endpoint.getAuth(), - endpoint.getDb(), - endpoint.getTimeout()), - sPos, - ePos, - i, - total)); - } - } - return result.toArray(new RedisNode[0]); - } finally { - conn.close(); - } - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/RedisUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/RedisUtils.java deleted file mode 100644 index 21fc775f8b..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/RedisUtils.java +++ /dev/null @@ -1,479 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -import org.apache.streampark.shaded.org.slf4j.Logger; - -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisCluster; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.ScanParams; -import redis.clients.jedis.Transaction; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; - -/** Redis command utilities. */ -public final class RedisUtils { - - private static final Logger LOG = - StreamParkLoggerFactory.loggerFactory().getLogger(RedisUtils.class.getName()); - - private RedisUtils() { - } - - public static boolean exists(String key, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.exists(key), endpoint); - } - - public static boolean hexists(String key, String field, RedisEndpoint endpoint) { - return hexists(key, field, null, endpoint); - } - - public static boolean hexists(String key, String field, Runnable func, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.hexists(key, field), func, endpoint); - } - - public static String get(String key, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.get(key), endpoint); - } - - public static String hget(String key, String field, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.hget(key, field), endpoint); - } - - public static String setex(String key, String value, Integer ttl, RedisEndpoint endpoint) { - return setex(key, value, ttl, null, endpoint); - } - - public static String setex(String key, String value, Integer ttl, Runnable func, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.setex(key, ttl, value), func, endpoint); - } - - public static Long setnx(String key, String value, Integer ttl, RedisEndpoint endpoint) { - return setnx(key, value, ttl, null, endpoint); - } - - public static Long setnx(String key, String value, Integer ttl, Runnable func, RedisEndpoint endpoint) { - return doRedis( - jedis -> { - Long x = jedis.setnx(key, value); - if (x == 1 && ttl != null) { - jedis.expire(key, ttl); - } - return x; - }, - func, - endpoint); - } - - public static Long hsetnx( - String key, String field, String value, Integer ttl, RedisEndpoint endpoint) { - return hsetnx(key, field, value, ttl, null, endpoint); - } - - public static Long hsetnx( - String key, String field, String value, Integer ttl, Runnable func, - RedisEndpoint endpoint) { - return doRedis( - jedis -> { - Long x = jedis.hsetnx(key, field, value); - if (x == 1 && ttl != null) { - jedis.expire(key, ttl); - } - return x; - }, - func, - endpoint); - } - - public static String[] mget(String[] keys, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.mget(keys).toArray(new String[0]), endpoint); - } - - public static Long del(String key, RedisEndpoint endpoint) { - return del(key, null, endpoint); - } - - public static Long del(String key, Runnable func, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.del(key), func, endpoint); - } - - public static String set(String key, String value, Integer ttl, RedisEndpoint endpoint) { - return set(key, value, ttl, null, endpoint); - } - - public static String set(String key, String value, Integer ttl, Runnable func, RedisEndpoint endpoint) { - return doRedis( - jedis -> { - String s = jedis.set(key, value); - if (ttl != null) { - jedis.expire(key, ttl); - } - return s; - }, - func, - endpoint); - } - - public static Long hset(String key, String field, String value, Integer ttl, RedisEndpoint endpoint) { - return hset(key, field, value, ttl, null, endpoint); - } - - public static Long hset( - String key, String field, String value, Integer ttl, Runnable func, - RedisEndpoint endpoint) { - return doRedis( - jedis -> { - Long s = jedis.hset(key, field, value); - if (ttl != null) { - jedis.expire(key, ttl); - } - return s; - }, - func, - endpoint); - } - - public static String hmset( - String key, Map hash, Integer ttl, RedisEndpoint endpoint) { - return hmset(key, hash, ttl, null, endpoint); - } - - public static String hmset( - String key, Map hash, Integer ttl, Runnable func, - RedisEndpoint endpoint) { - return doRedis( - jedis -> { - String s = jedis.hmset(key, hash); - if (ttl != null) { - jedis.expire(key, ttl); - } - return s; - }, - func, - endpoint); - } - - public static List hmget(String key, String[] fields, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.hmget(key, fields), endpoint); - } - - public static Map hgetAll(String key, RedisEndpoint endpoint) { - return doRedis(jedis -> new HashMap<>(jedis.hgetAll(key)), endpoint); - } - - public static Long hdel(String key, List fields, RedisEndpoint endpoint) { - return hdel(key, fields, null, endpoint); - } - - public static Long hdel(String key, List fields, Runnable func, RedisEndpoint endpoint) { - if (key == null || fields == null || fields.isEmpty()) { - return 0L; - } - return doRedis(jedis -> jedis.hdel(key, fields.toArray(new String[0])), func, endpoint); - } - - public static Long sadd(String key, List members, Integer ttl, RedisEndpoint endpoint) { - return sadd(key, members, ttl, null, endpoint); - } - - public static Long sadd( - String key, List members, Integer ttl, Runnable func, RedisEndpoint endpoint) { - return doRedis( - jedis -> { - Long res = jedis.sadd(key, members.toArray(new String[0])); - if (ttl != null) { - jedis.expire(key, ttl); - } - return res; - }, - func, - endpoint); - } - - public static Set smembers(String key, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.smembers(key), endpoint); - } - - public static Long srem(String key, List members, RedisEndpoint endpoint) { - return srem(key, members, null, endpoint); - } - - public static Long srem(String key, List members, Runnable func, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.srem(key, members.toArray(new String[0])), func, endpoint); - } - - public static String getOrElseHset( - String key, String field, String value, Integer ttl, RedisEndpoint endpoint) { - return getOrElseHset(key, field, value, ttl, null, endpoint); - } - - public static String getOrElseHset( - String key, - String field, - String value, - Integer ttl, - Runnable func, - RedisEndpoint endpoint) { - return doRedis( - jedis -> { - String v = jedis.hget(key, field); - if (v == null) { - jedis.hset(key, field, value); - if (ttl != null) { - jedis.expire(key, ttl); - } - } - return v; - }, - func, - endpoint); - } - - public static String getOrElseSet(String key, String value, Integer ttl, RedisEndpoint endpoint) { - return getOrElseSet(key, value, ttl, null, endpoint); - } - - public static String getOrElseSet( - String key, String value, Integer ttl, Runnable func, RedisEndpoint endpoint) { - return doRedis( - jedis -> { - String v = jedis.get(key); - if (v == null) { - jedis.set(key, value); - if (ttl != null) { - jedis.expire(key, ttl); - } - } - return v; - }, - func, - endpoint); - } - - public static Long hincrBy( - String key, String field, long value, Integer ttl, RedisEndpoint endpoint) { - return hincrBy(key, field, value, ttl, null, endpoint); - } - - public static Long hincrBy( - String key, String field, long value, Integer ttl, Runnable func, - RedisEndpoint endpoint) { - return doRedis( - jedis -> { - Long reply = jedis.hincrBy(key, field, value); - if (ttl != null) { - jedis.expire(key, ttl); - } - return reply; - }, - func, - endpoint); - } - - public static Double hincrByFloat( - String key, String field, double value, Integer ttl, RedisEndpoint endpoint) { - return hincrByFloat(key, field, value, ttl, null, endpoint); - } - - public static Double hincrByFloat( - String key, String field, double value, Integer ttl, Runnable func, - RedisEndpoint endpoint) { - return doRedis( - jedis -> { - Double reply = jedis.hincrByFloat(key, field, value); - if (ttl != null) { - jedis.expire(key, ttl); - } - return reply; - }, - func, - endpoint); - } - - public static Long incrBy(String key, long value, Integer ttl, RedisEndpoint endpoint) { - return incrBy(key, value, ttl, null, endpoint); - } - - public static Long incrBy( - String key, long value, Integer ttl, Runnable func, RedisEndpoint endpoint) { - return doRedis( - jedis -> { - Long reply = jedis.incrBy(key, value); - if (ttl != null) { - jedis.expire(key, ttl); - } - return reply; - }, - func, - endpoint); - } - - public static Double incrByFloat(String key, double value, Integer ttl, RedisEndpoint endpoint) { - return incrByFloat(key, value, ttl, null, endpoint); - } - - public static Double incrByFloat( - String key, double value, Integer ttl, Runnable func, RedisEndpoint endpoint) { - return doRedis( - jedis -> { - Double reply = jedis.incrByFloat(key, value); - if (ttl != null) { - jedis.expire(key, ttl); - } - return reply; - }, - func, - endpoint); - } - - public static Long mSets(List> kvs, Integer ttl, RedisEndpoint endpoint) { - return mSets(kvs, ttl, null, endpoint); - } - - public static Long mSets( - List> kvs, Integer ttl, Runnable func, RedisEndpoint endpoint) { - return doRedis( - jedis -> { - long start = System.currentTimeMillis(); - Pipeline pipe = jedis.pipelined(); - for (Map.Entry kv : kvs) { - pipe.mset(kv.getKey(), kv.getValue()); - if (ttl != null) { - pipe.expire(kv.getKey(), ttl); - } - } - pipe.sync(); - return System.currentTimeMillis() - start; - }, - func, - endpoint); - } - - public static Long mSetex(List> kvs, Integer ttl, RedisEndpoint endpoint) { - return mSetex(kvs, ttl, null, endpoint); - } - - public static Long mSetex( - List> kvs, Integer ttl, Runnable func, RedisEndpoint endpoint) { - return doRedis( - jedis -> { - long start = System.currentTimeMillis(); - Pipeline pipe = jedis.pipelined(); - for (Map.Entry kv : kvs) { - pipe.setnx(kv.getKey(), kv.getValue()); - if (ttl != null) { - pipe.expire(kv.getKey(), ttl); - } - } - pipe.sync(); - return System.currentTimeMillis() - start; - }, - func, - endpoint); - } - - public static Long expire(String key, int seconds, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.expire(key, seconds), endpoint); - } - - public static void delByPattern(String key, RedisEndpoint endpoint) { - delByPattern(key, null, endpoint); - } - - public static void delByPattern(String key, Runnable func, RedisEndpoint endpoint) { - doRedis( - jedis -> { - ScanParams scanParams = new ScanParams(); - scanParams.match(key); - scanParams.count(10000); - String cursor = ScanParams.SCAN_POINTER_START; - do { - redis.clients.jedis.ScanResult scanResult = jedis.scan(cursor, scanParams); - cursor = scanResult.getCursor(); - List keys = scanResult.getResult(); - if (!keys.isEmpty()) { - jedis.del(keys.toArray(new String[0])); - } - } while (!"0".equals(cursor)); - return null; - }, - func, - endpoint); - } - - public static Long hlen(String key, RedisEndpoint endpoint) { - return doRedis(jedis -> jedis.hlen(key), endpoint); - } - - public static R doRedis(Function f, RedisEndpoint endpoint) { - return doRedis(f, null, endpoint); - } - - public static R doRedis(Function f, Runnable func, RedisEndpoint endpoint) { - Jedis redis = RedisClient.connect(endpoint); - R result; - if (func == null) { - result = f.apply(redis); - } else { - Transaction transaction = redis.multi(); - result = f.apply(redis); - func.run(); - transaction.exec(); - transaction.close(); - } - try { - redis.close(); - LOG.debug("[StreamPark] jedis.close successful."); - } catch (Exception e) { - LOG.error("[StreamPark] jedis.close failed."); - } - return result; - } - - public static R doCluster(Function f, RedisEndpoint... endpoints) { - JedisCluster cluster = RedisClient.connectCluster(endpoints); - R result = f.apply(cluster); - try { - cluster.close(); - LOG.debug("[StreamPark] cluster.close successful."); - } catch (Exception e) { - LOG.error("[StreamPark] cluster.close failed."); - } - return result; - } - - public static R doPipeline(Function f, RedisEndpoint endpoint) { - Jedis redis = RedisClient.connect(endpoint); - Pipeline pipe = redis.pipelined(); - R result = f.apply(pipe); - try { - pipe.sync(); - pipe.close(); - redis.close(); - LOG.debug("[StreamPark] pipe.close successful."); - } catch (Exception e) { - LOG.error("[StreamPark] pipe.close failed."); - } - return result; - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/SqlConvertUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/SqlConvertUtils.java deleted file mode 100644 index e711763430..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/SqlConvertUtils.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -import org.apache.streampark.shaded.org.slf4j.Logger; - -import java.util.AbstractMap; -import java.util.HashMap; -import java.util.Map; -import java.util.Scanner; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** SQL dialect conversion utilities. */ -public final class SqlConvertUtils { - - private static final Logger LOG = - StreamParkLoggerFactory.loggerFactory().getLogger(SqlConvertUtils.class.getName()); - - private static final Pattern FIELD_REGEXP = - Pattern.compile( - "\\s*(.*?)\\s+(([a-z]+)\\((.*?)\\)|[a-z]+)(\\s*|((.*?)(comment)\\s+(['|\"](.*?)['|\"])|(.*?))),$", - Pattern.CASE_INSENSITIVE); - - private static final Pattern PRIMARY_REGEXP = - Pattern.compile("primary\\s+key\\s+\\((.*?)\\)", Pattern.CASE_INSENSITIVE); - - private static final Pattern COMMENT_REGEXP = - Pattern.compile("(comment)\\s+(['\"])", Pattern.CASE_INSENSITIVE); - - private static final Pattern LENGTH_REGEXP = Pattern.compile("(.*?)\\s*\\([^\\\\)|^\\n]+,$"); - - private SqlConvertUtils() { - } - - private static String toFlinkDataType(String dataType, String length) { - switch (dataType.toUpperCase()) { - case "TEXT": - case "LONGTEXT": - return "VARCHAR"; - case "DATETIME": - return "TIMESTAMP"; - case "INT": - return "INTEGER"; - default: - return dataType.toUpperCase(); - } - } - - private static String toClickhouseDataType(String dataType, String length) { - switch (dataType.toUpperCase()) { - case "TEXT": - case "LONGTEXT": - case "BLOB": - case "VARCHAR": - case "VARBINARY": - return "String"; - case "DATETIME": - case "TIMESTAMP": - return "DateTime"; - case "TINYINT": - return "Int8"; - case "SMALLINT": - return "Int16"; - case "FLOAT": - return "Float32"; - case "DOUBLE": - return "Float64"; - case "DATE": - return "Date"; - case "BIGINT": - case "INT": - if (length == null) { - return "Int32"; - } - int len = Integer.parseInt(length.trim()); - if (len < 3) { - return "Int8"; - } else if (len < 5) { - return "Int16"; - } else if (len < 9) { - return "Int32"; - } - return "Int64"; - case "DECIMAL": - String[] parts = length.split(","); - if (parts.length == 2) { - int p = Integer.parseInt(parts[0].trim()); - int s = Integer.parseInt(parts[1].trim()); - int sum = p + s; - if (sum <= 32) { - return "Decimal32(" + s + ")"; - } else if (sum <= 64) { - return "Decimal64(" + s + ")"; - } - return "Decimal128(" + s + ")"; - } - throw new IllegalArgumentException("Invalid decimal length: " + length); - default: - return dataType.toUpperCase(); - } - } - - private static String formatSql(String sql) { - String body = - sql.substring(sql.indexOf('('), sql.lastIndexOf(')') + 1) - .replaceAll("\r|\n|\r\n", "") - .replaceFirst("\\(", "(\n") - .replaceFirst("\\)$", "\n)") - .replaceAll(",", ",\n"); - - Scanner scanner = new Scanner(body); - Map map = new HashMap<>(); - while (scanner.hasNextLine()) { - map.put(map.size(), scanner.nextLine().trim()); - } - StringBuilder sqlBuffer = new StringBuilder(sql.substring(0, sql.indexOf('('))); - int skipNo = -1; - for (Map.Entry entry : map.entrySet()) { - int idx = entry.getKey(); - if (idx > skipNo) { - AbstractMap.SimpleEntry length = lengthJoin(map, idx, entry.getValue()); - if (length.getKey() > idx) { - sqlBuffer.append(length.getValue()).append('\n'); - skipNo = length.getKey(); - } else { - AbstractMap.SimpleEntry comment = commentJoin(map, idx, entry.getValue()); - if (comment.getKey() > idx) { - sqlBuffer.append(comment.getValue()).append('\n'); - skipNo = comment.getKey(); - } else { - sqlBuffer.append(entry.getValue()).append('\n'); - } - } - } - } - scanner.close(); - return sqlBuffer.toString().trim().concat(sql.substring(sql.lastIndexOf(')') + 1)); - } - - private static AbstractMap.SimpleEntry commentJoin( - Map map, int index, - String segment) { - Matcher matcher = COMMENT_REGEXP.matcher(segment); - if (!matcher.find()) { - return new AbstractMap.SimpleEntry<>(index, segment); - } - String quote = matcher.group(2); - String regexp = "\\" + quote + "(,|)$"; - String cleaned = - segment.replaceFirst("(?i)(comment)\\s+(['\"])", "").replace("\\" + quote, ""); - if (cleaned.matches(".*" + regexp)) { - return new AbstractMap.SimpleEntry<>(index, segment); - } - String nextLine = map.get(index + 1); - return commentJoin(map, index + 1, segment + nextLine); - } - - private static AbstractMap.SimpleEntry lengthJoin( - Map map, int index, - String segment) { - if (!LENGTH_REGEXP.matcher(segment).find()) { - return commentJoin(map, index, segment); - } - String nextLine = map.get(index + 1); - return lengthJoin(map, index + 1, segment.trim() + nextLine); - } - - private interface TypeConverter { - - String convert(String dataType, String length); - } - - private interface KeyConverter { - - String convert(String line); - } - - private static String convertSql( - String sql, TypeConverter typeFunc, KeyConverter keyFunc, String postfix) { - String formattedSql = formatSql(sql); - Scanner scanner = new Scanner(formattedSql); - StringBuilder sqlBuffer = new StringBuilder(); - while (scanner.hasNextLine()) { - String rawLine = scanner.nextLine().trim(); - String line = null; - String upper = rawLine.toUpperCase().trim(); - if (upper.startsWith("CREATE ")) { - line = rawLine; - } else if (upper.startsWith("PRIMARY KEY ")) { - if (keyFunc != null) { - line = keyFunc.convert(rawLine); - } - } else if (!upper.startsWith("UNIQUE KEY ") && !upper.startsWith("KEY ")) { - Matcher matcher = FIELD_REGEXP.matcher(rawLine); - if (matcher.find()) { - String fieldName = matcher.group(1); - String dataType; - if (matcher.group(3) != null) { - dataType = matcher.group(3); - } else { - dataType = matcher.group(2); - } - String length = matcher.group(4); - if (dataType != null) { - String fieldType = typeFunc.convert(dataType, length); - if (matcher.group(8) != null) { - line = fieldName + " " + fieldType + " COMMENT '" + matcher.group(10) + "'"; - } else { - line = fieldName + " " + fieldType; - } - } - } - } - if (line != null) { - sqlBuffer.append(line); - if (line.toUpperCase().trim().startsWith("CREATE ")) { - sqlBuffer.append('\n'); - } else { - sqlBuffer.append(",\n"); - } - } - } - scanner.close(); - String result = sqlBuffer.toString().trim().replaceAll(",$", "\n)"); - if (postfix != null) { - return result + " " + postfix; - } - return result; - } - - public static String mysqlToFlinkSql(String sql, String postfix) { - return convertSql( - sql, - SqlConvertUtils::toFlinkDataType, - line -> { - Matcher matcher = PRIMARY_REGEXP.matcher(line); - if (matcher.find()) { - return matcher.group() + " NOT ENFORCED"; - } - return null; - }, - postfix); - } - - public static String mysqlToClickhouse(String sql, String postfix) { - return convertSql(sql, SqlConvertUtils::toClickhouseDataType, null, postfix); - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/StringCastUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/StringCastUtils.java deleted file mode 100644 index 55e24ee6e1..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/StringCastUtils.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -/** String to primitive/boxed type conversion utilities. */ -public final class StringCastUtils { - - private StringCastUtils() { - } - - @SuppressWarnings("unchecked") - public static T cast(String v, Class classType) { - return (T) classType.cast(parseValue(v, classType)); - } - - private static Object parseValue(String v, Class classType) { - if (classType == String.class) { - return v; - } - if (classType == Byte.class || classType == byte.class) { - return Byte.parseByte(v); - } - if (classType == Integer.class || classType == int.class) { - return Integer.parseInt(v); - } - if (classType == Long.class || classType == long.class) { - return Long.parseLong(v); - } - if (classType == Float.class || classType == float.class) { - return Float.parseFloat(v); - } - if (classType == Double.class || classType == double.class) { - return Double.parseDouble(v); - } - if (classType == Short.class || classType == short.class) { - return Short.parseShort(v); - } - if (classType == Boolean.class || classType == boolean.class) { - return Boolean.parseBoolean(v); - } - throw new IllegalArgumentException("Unsupported type: " + classType); - } -} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple.java b/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple.java new file mode 100644 index 0000000000..5652c03700 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import java.io.Serializable; + +/** + * The base class of all tuples. Tuples have a fix length and contain a set of fields, which may all + * be of different types. Because Tuples are strongly typed, each distinct tuple length is + * represented by its own class. Tuples exists with up to 25 fields and are described in the classes + * {@link Tuple1} to {@link Tuple4}. + * + *

The fields in the tuples may be accessed directly a public fields, or via position (zero + * indexed) {@link #get(int)}. + * + *

Tuples are in principle serializable. However, they may contain non-serializable fields, in + * which case serialization will fail. + */ +public abstract class Tuple implements Serializable { + + private static final long serialVersionUID = 1L; + + /** + * Gets the field at the specified position. + * + * @param pos The position of the field, zero indexed. + * @return The field at the specified position. + * @throws IndexOutOfBoundsException Thrown, if the position is negative, or equal to, or larger + * than the number of fields. + */ + public abstract T get(int pos); + + /** + * Sets the field at the specified position. + * + * @param value The value to be assigned to the field at the specified position. + * @param pos The position of the field, zero indexed. + * @throws IndexOutOfBoundsException Thrown, if the position is negative, or equal to, or larger + * than the number of fields. + */ + public abstract void set(T value, int pos); + + /** + * Shallow tuple copy. + * + * @return A new Tuple with the same fields as this. + */ + public abstract T copy(); +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple1.java b/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple1.java new file mode 100644 index 0000000000..0f4f980a40 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple1.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import java.util.Objects; + +/** + * Mutable single-field tuple with positional and direct field access. + * + * @param field type + */ +public class Tuple1 extends Tuple { + + private static final long serialVersionUID = 1L; + + /** Field 0 of the tuple. */ + public T0 _1; + + /** Creates a new tuple where all fields are null. */ + public Tuple1() { + } + + /** + * Creates a new tuple and assigns the given values to the tuple's fields. + * + * @param t0 The value for field 0 + */ + public Tuple1(T0 t0) { + this._1 = t0; + } + + @Override + @SuppressWarnings("unchecked") + public T get(int pos) { + if (pos == 0) { + return (T) this._1; + } + throw new IndexOutOfBoundsException(String.valueOf(pos)); + } + + @Override + @SuppressWarnings("unchecked") + public void set(T value, int pos) { + if (pos == 0) { + this._1 = (T0) value; + } else { + throw new IndexOutOfBoundsException(String.valueOf(pos)); + } + } + + /** + * Sets new values to all fields of the tuple. + * + * @param f0 The value for field 0 + */ + public void set(T0 f0) { + this._1 = f0; + } + + /** + * Deep equality for tuples by calling equals() on the tuple members. + * + * @param o the object checked for equality + * @return true if this is equal to o. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Tuple1)) { + return false; + } + @SuppressWarnings("rawtypes") + Tuple1 tuple = (Tuple1) o; + return Objects.equals(_1, tuple._1); + } + + @Override + public int hashCode() { + return _1 != null ? _1.hashCode() : 0; + } + + /** + * Shallow tuple copy. + * + * @return A new Tuple with the same fields as this. + */ + @Override + @SuppressWarnings("unchecked") + public Tuple1 copy() { + return new Tuple1<>(this._1); + } + + /** + * Creates a new tuple and assigns the given values to the tuple's fields. This is more convenient + * than using the constructor, because the compiler can infer the generic type arguments + * implicitly. For example: {@code Tuple3.of(n, x, s)} instead of {@code new Tuple3(n, x, s)} + */ + public static Tuple1 of(T0 f0) { + return new Tuple1<>(f0); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple2.java b/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple2.java new file mode 100644 index 0000000000..9aa923e352 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple2.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import java.util.Objects; + +/** + * A tuple with 2 fields. Tuples are strongly typed; each field may be of a separate type. The + * fields of the tuple can be accessed directly as public fields (f0, f1, ...) or via their position + * through the {@link #get(int)} method. The tuple field positions start at zero. + * + *

Tuples are mutable types, meaning that their fields can be re-assigned. This allows functions + * that work with Tuples to reuse objects in order to reduce pressure on the garbage collector. + * + *

Warning: If you subclass Tuple2, then be sure to either + * + *

    + *
  • not add any new fields, or + *
  • make it a POJO, and always declare the element type of your DataStreams/DataSets to your + * descendant type. (That is, if you have a "class Foo extends Tuple2", then don't use + * instances of Foo in a DataStream<Tuple2> / DataSet<Tuple2>, but declare it as + * DataStream<Foo> / DataSet<Foo>.) + *
+ * + * @see Tuple + * @param The type of field 0 + * @param The type of field 1 + */ +public class Tuple2 extends Tuple { + + private static final long serialVersionUID = 1L; + + /** Field 0 of the tuple. */ + public T0 _1; + /** Field 1 of the tuple. */ + public T1 _2; + + /** Creates a new tuple where all fields are null. */ + public Tuple2() { + } + + /** + * Creates a new tuple and assigns the given values to the tuple's fields. + * + * @param t0 The value for field 0 + * @param _1 The value for field 1 + */ + public Tuple2(T0 t0, T1 _1) { + this._1 = t0; + this._2 = _1; + } + + @Override + @SuppressWarnings("unchecked") + public T get(int pos) { + switch (pos) { + case 0: + return (T) this._1; + case 1: + return (T) this._2; + default: + throw new IndexOutOfBoundsException(String.valueOf(pos)); + } + } + + @SuppressWarnings("unchecked") + public void set(T value, int pos) { + switch (pos) { + case 0: + this._1 = (T0) value; + break; + case 1: + this._2 = (T1) value; + break; + default: + throw new IndexOutOfBoundsException(String.valueOf(pos)); + } + } + + /** + * Sets new values to all fields of the tuple. + * + * @param f0 The value for field 0 + * @param f1 The value for field 1 + */ + public void set(T0 f0, T1 f1) { + this._1 = f0; + this._2 = f1; + } + + /** + * Returns a shallow copy of the tuple with swapped values. + * + * @return shallow copy of the tuple with swapped values + */ + public Tuple2 swap() { + return new Tuple2(_2, _1); + } + + /** + * Deep equality for tuples by calling equals() on the tuple members. + * + * @param o the object checked for equality + * @return true if this is equal to o. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Tuple2)) { + return false; + } + @SuppressWarnings("rawtypes") + Tuple2 tuple = (Tuple2) o; + if (!Objects.equals(_1, tuple._1)) { + return false; + } + return Objects.equals(_2, tuple._2); + } + + @Override + public int hashCode() { + int result = _1 != null ? _1.hashCode() : 0; + result = 31 * result + (_2 != null ? _2.hashCode() : 0); + return result; + } + + /** + * Shallow tuple copy. + * + * @return A new Tuple with the same fields as this. + */ + @Override + @SuppressWarnings("unchecked") + public Tuple2 copy() { + return new Tuple2<>(this._1, this._2); + } + + /** + * Creates a new tuple and assigns the given values to the tuple's fields. This is more convenient + * than using the constructor, because the compiler can infer the generic type arguments + * implicitly. For example: {@code Tuple3.of(n, x, s)} instead of {@code new Tuple3(n, x, s)} + */ + public static Tuple2 of(T0 f0, T1 f1) { + return new Tuple2<>(f0, f1); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple3.java b/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple3.java new file mode 100644 index 0000000000..55c32947c2 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple3.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import java.util.Objects; + +/** + * A tuple with 3 fields. Tuples are strongly typed; each field may be of a separate type. The + * fields of the tuple can be accessed directly as public fields (f0, f1, ...) or via their position + * through the {@link #get(int)} method. The tuple field positions start at zero. + * + *

Tuples are mutable types, meaning that their fields can be re-assigned. This allows functions + * that work with Tuples to reuse objects in order to reduce pressure on the garbage collector. + * + *

Warning: If you subclass Tuple3, then be sure to either + * + *

    + *
  • not add any new fields, or + *
  • make it a POJO, and always declare the element type of your DataStreams/DataSets to your + * descendant type. (That is, if you have a "class Foo extends Tuple3", then don't use + * instances of Foo in a DataStream<Tuple3> / DataSet<Tuple3>, but declare it as + * DataStream<Foo> / DataSet<Foo>.) + *
+ * + * @see Tuple + * @param The type of field 0 + * @param The type of field 1 + * @param The type of field 2 + */ +public class Tuple3 extends Tuple { + + private static final long serialVersionUID = 1L; + + /** Field 0 of the tuple. */ + public T0 _1; + /** Field 1 of the tuple. */ + public T1 _2; + /** Field 2 of the tuple. */ + public T2 _3; + + /** Creates a new tuple where all fields are null. */ + public Tuple3() { + } + + /** + * Creates a new tuple and assigns the given values to the tuple's fields. + * + * @param t0 The value for field 0 + * @param t1 The value for field 1 + * @param t2 The value for field 2 + */ + public Tuple3(T0 t0, T1 t1, T2 t2) { + this._1 = t0; + this._2 = t1; + this._3 = t2; + } + + @Override + @SuppressWarnings("unchecked") + public T get(int pos) { + switch (pos) { + case 0: + return (T) this._1; + case 1: + return (T) this._2; + case 2: + return (T) this._3; + default: + throw new IndexOutOfBoundsException(String.valueOf(pos)); + } + } + + @Override + @SuppressWarnings("unchecked") + public void set(T value, int pos) { + switch (pos) { + case 0: + this._1 = (T0) value; + break; + case 1: + this._2 = (T1) value; + break; + case 2: + this._3 = (T2) value; + break; + default: + throw new IndexOutOfBoundsException(String.valueOf(pos)); + } + } + + /** + * Sets new values to all fields of the tuple. + * + * @param f0 The value for field 0 + * @param f1 The value for field 1 + * @param f2 The value for field 2 + */ + public void set(T0 f0, T1 f1, T2 f2) { + this._1 = f0; + this._2 = f1; + this._3 = f2; + } + + /** + * Deep equality for tuples by calling equals() on the tuple members. + * + * @param o the object checked for equality + * @return true if this is equal to o. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Tuple3)) { + return false; + } + @SuppressWarnings("rawtypes") + Tuple3 tuple = (Tuple3) o; + if (!Objects.equals(_1, tuple._1)) { + return false; + } + if (!Objects.equals(_2, tuple._2)) { + return false; + } + return Objects.equals(_3, tuple._3); + } + + @Override + public int hashCode() { + int result = _1 != null ? _1.hashCode() : 0; + result = 31 * result + (_2 != null ? _2.hashCode() : 0); + result = 31 * result + (_3 != null ? _3.hashCode() : 0); + return result; + } + + /** + * Shallow tuple copy. + * + * @return A new Tuple with the same fields as this. + */ + @Override + @SuppressWarnings("unchecked") + public Tuple3 copy() { + return new Tuple3<>(this._1, this._2, this._3); + } + + /** + * Creates a new tuple and assigns the given values to the tuple's fields. This is more convenient + * than using the constructor, because the compiler can infer the generic type arguments + * implicitly. For example: {@code Tuple3.of(n, x, s)} instead of {@code new Tuple3(n, x, s)} + */ + public static Tuple3 of(T0 f0, T1 f1, T2 f2) { + return new Tuple3<>(f0, f1, f2); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple4.java b/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple4.java new file mode 100644 index 0000000000..ab84261809 --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/Tuple4.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import java.util.Objects; + +/** + * A tuple with 4 fields. Tuples are strongly typed; each field may be of a separate type. The + * fields of the tuple can be accessed directly as public fields (f0, f1, ...) or via their position + * through the {@link #get(int)} method. The tuple field positions start at zero. + * + *

Tuples are mutable types, meaning that their fields can be re-assigned. This allows functions + * that work with Tuples to reuse objects in order to reduce pressure on the garbage collector. + * + *

Warning: If you subclass Tuple4, then be sure to either + * + *

    + *
  • not add any new fields, or + *
  • make it a POJO, and always declare the element type of your DataStreams/DataSets to your + * descendant type. (That is, if you have a "class Foo extends Tuple4", then don't use + * instances of Foo in a DataStream<Tuple4> / DataSet<Tuple4>, but declare it as + * DataStream<Foo> / DataSet<Foo>.) + *
+ * + * @see Tuple + * @param The type of field 0 + * @param The type of field 1 + * @param The type of field 2 + * @param The type of field 3 + */ +public class Tuple4 extends Tuple { + + private static final long serialVersionUID = 1L; + + /** Field 0 of the tuple. */ + public T0 _1; + /** Field 1 of the tuple. */ + public T1 _2; + /** Field 2 of the tuple. */ + public T2 _3; + /** Field 3 of the tuple. */ + public T3 _4; + + /** Creates a new tuple where all fields are null. */ + public Tuple4() { + } + + /** + * Creates a new tuple and assigns the given values to the tuple's fields. + * + * @param t0 The value for field 0 + * @param t1 The value for field 1 + * @param t2 The value for field 2 + * @param t3 The value for field 3 + */ + public Tuple4(T0 t0, T1 t1, T2 t2, T3 t3) { + this._1 = t0; + this._2 = t1; + this._3 = t2; + this._4 = t3; + } + + @Override + @SuppressWarnings("unchecked") + public T get(int pos) { + switch (pos) { + case 0: + return (T) this._1; + case 1: + return (T) this._2; + case 2: + return (T) this._3; + case 3: + return (T) this._4; + default: + throw new IndexOutOfBoundsException(String.valueOf(pos)); + } + } + + @Override + @SuppressWarnings("unchecked") + public void set(T value, int pos) { + switch (pos) { + case 0: + this._1 = (T0) value; + break; + case 1: + this._2 = (T1) value; + break; + case 2: + this._3 = (T2) value; + break; + case 3: + this._4 = (T3) value; + break; + default: + throw new IndexOutOfBoundsException(String.valueOf(pos)); + } + } + + /** + * Sets new values to all fields of the tuple. + * + * @param f0 The value for field 0 + * @param f1 The value for field 1 + * @param f2 The value for field 2 + * @param f3 The value for field 3 + */ + public void set(T0 f0, T1 f1, T2 f2, T3 f3) { + this._1 = f0; + this._2 = f1; + this._3 = f2; + this._4 = f3; + } + + /** + * Deep equality for tuples by calling equals() on the tuple members. + * + * @param o the object checked for equality + * @return true if this is equal to o. + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Tuple4)) { + return false; + } + @SuppressWarnings("rawtypes") + Tuple4 tuple = (Tuple4) o; + if (!Objects.equals(_1, tuple._1)) { + return false; + } + if (!Objects.equals(_2, tuple._2)) { + return false; + } + if (!Objects.equals(_3, tuple._3)) { + return false; + } + return Objects.equals(_4, tuple._4); + } + + @Override + public int hashCode() { + int result = _1 != null ? _1.hashCode() : 0; + result = 31 * result + (_2 != null ? _2.hashCode() : 0); + result = 31 * result + (_3 != null ? _3.hashCode() : 0); + result = 31 * result + (_4 != null ? _4.hashCode() : 0); + return result; + } + + /** + * Shallow tuple copy. + * + * @return A new Tuple with the same fields as this. + */ + @Override + @SuppressWarnings("unchecked") + public Tuple4 copy() { + return new Tuple4<>(this._1, this._2, this._3, this._4); + } + + /** + * Creates a new tuple and assigns the given values to the tuple's fields. This is more convenient + * than using the constructor, because the compiler can infer the generic type arguments + * implicitly. For example: {@code Tuple3.of(n, x, s)} instead of {@code new Tuple3(n, x, s)} + */ + public static Tuple4 of(T0 f0, T1 f1, T2 f2, T3 f3) { + return new Tuple4<>(f0, f1, f2, f3); + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java index 0fb5e294a2..902da5660b 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java @@ -85,7 +85,7 @@ public static String uuid() { public static void requireCheckJarFile(URL jar) throws IOException { try ( - InputStream in = SafePathUtils.openJarFile(jar); + InputStream in = PathUtils.openJarFile(jar); JarInputStream ignored = new JarInputStream(new BufferedInputStream(in))) { // verify jar is readable } catch (IOException e) { @@ -95,7 +95,7 @@ public static void requireCheckJarFile(URL jar) throws IOException { public static Manifest getJarManifest(File jarFile) throws IOException { try ( - InputStream in = SafePathUtils.openJarFile(jarFile.toURI().toURL()); + InputStream in = PathUtils.openJarFile(jarFile.toURI().toURL()); JarInputStream jarInputStream = new JarInputStream(new BufferedInputStream(in))) { return jarInputStream.getManifest(); } @@ -231,7 +231,7 @@ public static void printLogo(String info) { System.out.println(" ___/ / /_/ / / __/ /_/ / / / / / / /_/ / /_/ / / / ,< "); System.out.println(" /____/\\__/_/ \\___/\\__,_/_/ /_/ /_/ ____/\\__,_/_/ /_/|_| "); System.out.println(" /_/ \n\n"); - System.out.println(" Version: 3.0.0-SNAPSHOT "); + System.out.println(" Version: 3.0.0 "); System.out.println(" WebSite: https://streampark.apache.org "); System.out.println(" GitHub : https://github.com/apache/streampark "); System.out.println(" Info : " + info + " "); diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java b/streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java new file mode 100644 index 0000000000..cad042f6bc --- /dev/null +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java @@ -0,0 +1,489 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.streampark.common.configuration.ConfigException; + +import org.apache.streampark.shaded.org.slf4j.Logger; + +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.error.Mark; +import org.yaml.snakeyaml.error.MarkedYAMLException; + +import java.io.IOException; +import java.io.InputStream; +import java.io.Reader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Parses YAML configuration documents with strict defaults and a narrow compatibility fallback. + * + *

The parser first evaluates the document unchanged with SnakeYAML's safe constructor, + * duplicate-key rejection, and recursive-key rejection. Only a failed parse enables compatibility + * normalization for editor-style multiline scalar values and unquoted scalar values beginning + * with YAML-reserved characters. This ordering preserves standard YAML structures while accepting + * configuration text produced by StreamPark's existing editors. + * + *

Input streams are decoded from BOM markers when present and as UTF-8 otherwise. Parse errors + * intentionally omit source snippets because configuration lines can contain credentials. + * Parser and dumper instances are created per invocation, making this utility safe for concurrent + * configuration loading. + */ +public final class YamlParser { + + private static final Logger LOG = + StreamParkLoggerFactory.loggerFactory().getLogger(YamlParser.class.getName()); + + private static final String LENIENT_SCALAR_PREFIXES = "!&*%@`|>"; + private static final Pattern EMPTY_VALUE_PROPERTY = + Pattern.compile("^(\\s*)([^\\s#][^:]*):(?:\\s*(#.*)?)?$"); + private static final Pattern STRUCTURED_CHILD = + Pattern.compile("^[^#\\-][^:]*:(?:$|\\s.*)$"); + private static final Pattern BLOCK_SCALAR_HEADER = + Pattern.compile("^[|>](?:[+-]?\\d+|\\d+[+-]?|[+-])?$"); + + private YamlParser() { + } + + /** + * Parses a YAML file after decoding its byte-order marker, if present. + * + * @param path path to the YAML document + * @return the structured root mapping, or an empty map for an empty document + * @throws ConfigException when the file cannot be read or does not contain a valid mapping + */ + public static Map parse(Path path) { + Objects.requireNonNull(path, "path must not be null"); + try { + return parseBytes(Files.readAllBytes(path)); + } catch (IOException e) { + throw new ConfigException("Cannot read YAML configuration from " + path, e); + } + } + + /** + * Parses YAML bytes from a stream without closing it. + * + * @param inputStream stream containing a BOM-marked or UTF-8 document + * @return the structured root mapping, or an empty map for an empty document + * @throws ConfigException when the stream cannot be read or does not contain a valid mapping + */ + public static Map parse(InputStream inputStream) { + Objects.requireNonNull(inputStream, "inputStream must not be null"); + try { + return parseBytes(inputStream.readAllBytes()); + } catch (IOException e) { + throw new ConfigException("Cannot read YAML configuration stream", e); + } + } + + /** + * Parses YAML characters from a reader without closing it. + * + * @param reader reader containing a YAML document + * @return the structured root mapping, or an empty map for an empty document + * @throws ConfigException when the reader cannot be consumed or the document is invalid + */ + public static Map parse(Reader reader) { + Objects.requireNonNull(reader, "reader must not be null"); + try { + StringBuilder content = new StringBuilder(); + char[] buffer = new char[4096]; + int count; + while ((count = reader.read(buffer)) >= 0) { + content.append(buffer, 0, count); + } + return parse(content.toString()); + } catch (IOException e) { + throw new ConfigException("Cannot read YAML configuration", e); + } + } + + /** + * Parses YAML text into a structured mapping. + * + * @param text YAML document text + * @return the structured root mapping, or an empty map for an empty document + * @throws ConfigException when the document is invalid or its root is not a mapping + */ + public static Map parse(String text) { + Objects.requireNonNull(text, "YAML text must not be null"); + String source = stripBom(text); + try { + // Valid YAML must never be rewritten by compatibility rules. Parsing the original + // document first preserves standard mappings, sequences, tags, and block scalars. + return parseStrict(source); + } catch (RuntimeException originalFailure) { + String compatible = normalize(source); + if (!source.equals(compatible)) { + try { + Map result = parseStrict(compatible); + LOG.warn( + "YAML parsed after compatibility normalization; quote values that begin " + + "with YAML-reserved characters"); + return result; + } catch (RuntimeException compatibilityFailure) { + // The original failure describes user input; the fallback failure describes + // generated text and would point operators at content they never supplied. + } + } + throw safeError(originalFailure); + } + } + + /** + * Flattens a structured YAML mapping into dot-separated keys. + * + *

Lists remain structured values so the consuming configuration domain can choose its own + * serialization. Null leaves are omitted, and collisions between an explicit dotted key and a + * nested mapping fail fast. + * + * @param document structured YAML root mapping + * @return an immutable map containing dot-separated leaf keys + * @throws ConfigException when a key is invalid or flattened keys collide + */ + public static Map flatten(Map document) { + Objects.requireNonNull(document, "YAML document must not be null"); + Map result = new LinkedHashMap<>(); + flatten(document, "", result); + return Collections.unmodifiableMap(result); + } + + /** + * Serializes a structured value using compact YAML flow syntax without a trailing newline. + * + * @param value structured value + * @return compact YAML representation + * @throws ConfigException when the value cannot be represented safely + */ + public static String toFlowString(Object value) { + try { + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.FLOW); + options.setSplitLines(false); + String output = new Yaml(options).dump(value); + while (output.endsWith("\n") || output.endsWith("\r")) { + output = output.substring(0, output.length() - 1); + } + return output; + } catch (RuntimeException e) { + throw safeError(e); + } + } + + /** Applies the two compatibility repairs supported by StreamPark's configuration editor. */ + private static String normalize(String text) { + if (isBlank(text)) { + return text; + } + return quoteValues(foldLines(text)); + } + + /** + * Folds indented editor-style scalar continuation lines into the parent property. + * + *

Indented mappings, sequences, and block scalars are recognized as structured YAML and are + * never folded. + */ + private static String foldLines(String text) { + if (isBlank(text)) { + return text; + } + + String lineSeparator = text.contains("\r\n") ? "\r\n" : "\n"; + String[] lines = text.split("\\r?\\n", -1); + StringBuilder normalized = new StringBuilder(text.length()); + for (int index = 0; index < lines.length; index++) { + String currentLine = lines[index]; + Matcher matcher = EMPTY_VALUE_PROPERTY.matcher(currentLine); + if (!matcher.matches()) { + appendLine(normalized, currentLine, lineSeparator, index < lines.length - 1); + continue; + } + + int currentIndent = indent(currentLine); + int nextIndex = index + 1; + List scalarEntries = new ArrayList<>(); + boolean structuredChild = false; + + // A property without an inline value may either own nested YAML or contain plain text + // entered on following lines. Inspect the entire indented region before modifying it. + while (nextIndex < lines.length) { + String childLine = lines[nextIndex]; + if (isBlank(childLine)) { + nextIndex++; + continue; + } + if (indent(childLine) <= currentIndent) { + break; + } + String trimmedChild = childLine.trim(); + if (trimmedChild.startsWith("#")) { + nextIndex++; + continue; + } + if (isStructured(trimmedChild)) { + // Never partially fold a mapping or sequence. Leaving the original block intact + // allows the strict parser to retain its structure on the next attempt. + structuredChild = true; + break; + } + scalarEntries.add(trimmedChild); + nextIndex++; + } + + if (scalarEntries.isEmpty() || structuredChild) { + appendLine(normalized, currentLine, lineSeparator, index < lines.length - 1); + continue; + } + + normalized + .append(matcher.group(1)) + .append(matcher.group(2).trim()) + .append(": ") + .append(String.join(" ", scalarEntries)); + if (nextIndex < lines.length) { + normalized.append(lineSeparator); + } + index = nextIndex - 1; + } + return normalized.toString(); + } + + private static Map parseBytes(byte[] bytes) { + return parse(decode(bytes)); + } + + @SuppressWarnings("unchecked") + private static Map parseStrict(String text) { + LoaderOptions options = new LoaderOptions(); + options.setAllowDuplicateKeys(false); + options.setAllowRecursiveKeys(false); + Object result = new Yaml(new SafeConstructor(options)).load(text); + if (result == null) { + return new LinkedHashMap<>(); + } + if (!(result instanceof Map)) { + throw new ConfigException("YAML configuration root must be a mapping"); + } + return (Map) result; + } + + private static void flatten(Map source, String prefix, Map target) { + for (Map.Entry entry : source.entrySet()) { + if (!(entry.getKey() instanceof String)) { + throw new ConfigException("YAML configuration keys must be strings"); + } + String segment = ((String) entry.getKey()).trim(); + if (segment.isEmpty()) { + throw new ConfigException("YAML key segments must not be blank"); + } + String key = prefix.isEmpty() ? segment : prefix + "." + segment; + Object value = entry.getValue(); + if (value instanceof Map) { + flatten((Map) value, key, target); + } else if (value != null) { + if (target.putIfAbsent(key, value) != null) { + throw new ConfigException("Duplicate configuration key: " + key); + } + } + } + } + + private static String quoteValues(String text) { + String lineSeparator = text.contains("\r\n") ? "\r\n" : "\n"; + String[] lines = text.split("\\r?\\n", -1); + StringBuilder normalized = new StringBuilder(text.length()); + for (int index = 0; index < lines.length; index++) { + appendLine( + normalized, + quoteValue(lines[index]), + lineSeparator, + index < lines.length - 1); + } + return normalized.toString(); + } + + private static String quoteValue(String line) { + if (isBlank(line) || line.trim().startsWith("#")) { + return line; + } + int separatorIndex = line.indexOf(':'); + if (separatorIndex < 0 || separatorIndex >= line.length() - 1) { + return line; + } + int valueStart = separatorIndex + 1; + while (valueStart < line.length() && Character.isWhitespace(line.charAt(valueStart))) { + valueStart++; + } + if (valueStart >= line.length() + || LENIENT_SCALAR_PREFIXES.indexOf(line.charAt(valueStart)) < 0) { + return line; + } + + // Quote only the scalar portion. An inline comment must remain outside the generated quotes. + int valueEnd = valueEnd(line, valueStart); + String value = line.substring(valueStart, valueEnd).trim(); + // A standalone | or > (optionally with chomping/indent indicators) is valid block syntax. + if (value.isEmpty() || BLOCK_SCALAR_HEADER.matcher(value).matches()) { + return line; + } + return line.substring(0, valueStart) + + '\'' + + value.replace("'", "''") + + '\'' + + line.substring(valueEnd); + } + + private static int valueEnd(String line, int valueStart) { + for (int index = valueStart; index < line.length(); index++) { + if (line.charAt(index) == '#' + && index > valueStart + && Character.isWhitespace(line.charAt(index - 1))) { + int valueEnd = index; + while (valueEnd > valueStart + && Character.isWhitespace(line.charAt(valueEnd - 1))) { + valueEnd--; + } + return valueEnd; + } + } + return line.length(); + } + + private static boolean isStructured(String line) { + return "-".equals(line) + || line.startsWith("- ") + || STRUCTURED_CHILD.matcher(line).matches(); + } + + private static int indent(String line) { + int count = 0; + while (count < line.length() && Character.isWhitespace(line.charAt(count))) { + count++; + } + return count; + } + + private static boolean isBlank(String text) { + return text == null || text.trim().isEmpty(); + } + + private static String stripBom(String text) { + return !text.isEmpty() && text.charAt(0) == '\ufeff' ? text.substring(1) : text; + } + + private static String decode(byte[] bytes) { + if (bytes.length == 0) { + return ""; + } + // Check four-byte markers before UTF-16 because UTF-32LE starts with the UTF-16LE marker. + if (startsWith(bytes, 0x00, 0x00, 0xFE, 0xFF)) { + return decode(bytes, 4, Charset.forName("UTF-32BE")); + } + if (startsWith(bytes, 0xFF, 0xFE, 0x00, 0x00)) { + return decode(bytes, 4, Charset.forName("UTF-32LE")); + } + if (startsWith(bytes, 0xEF, 0xBB, 0xBF)) { + return decode(bytes, 3, StandardCharsets.UTF_8); + } + if (startsWith(bytes, 0xFE, 0xFF)) { + return decode(bytes, 2, StandardCharsets.UTF_16BE); + } + if (startsWith(bytes, 0xFF, 0xFE)) { + return decode(bytes, 2, StandardCharsets.UTF_16LE); + } + return new String(bytes, StandardCharsets.UTF_8); + } + + private static String decode(byte[] bytes, int offset, Charset charset) { + return new String(bytes, offset, bytes.length - offset, charset); + } + + private static boolean startsWith(byte[] bytes, int... prefix) { + if (bytes.length < prefix.length) { + return false; + } + for (int index = 0; index < prefix.length; index++) { + if ((bytes[index] & 0xFF) != prefix[index]) { + return false; + } + } + return true; + } + + private static void appendLine( + StringBuilder target, + String line, + String lineSeparator, + boolean appendLineBreak) { + target.append(line); + if (appendLineBreak) { + target.append(lineSeparator); + } + } + + private static ConfigException safeError(RuntimeException exception) { + if (!(exception instanceof MarkedYAMLException)) { + return exception instanceof ConfigException + ? (ConfigException) exception + : new ConfigException("Invalid YAML configuration"); + } + // MarkedYAMLException#getMessage includes the offending source line. Rebuild the diagnostic + // from problem and location fields so credentials in configuration values are never logged. + MarkedYAMLException marked = (MarkedYAMLException) exception; + StringBuilder message = new StringBuilder("Invalid YAML configuration"); + appendProblem(message, marked.getContext()); + appendMark(message, marked.getContextMark()); + appendProblem(message, marked.getProblem()); + appendMark(message, marked.getProblemMark()); + return new ConfigException(message.toString()); + } + + private static void appendProblem(StringBuilder message, String problem) { + if (problem != null && !problem.trim().isEmpty()) { + message.append(": ").append(problem.trim()); + } + } + + private static void appendMark(StringBuilder message, Mark mark) { + if (mark != null) { + message + .append(" in ") + .append(mark.getName()) + .append(" at line ") + .append(mark.getLine() + 1) + .append(", column ") + .append(mark.getColumn() + 1); + } + } +} diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/YarnUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/YarnUtils.java index 574528a1e0..441470f4c2 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/YarnUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/YarnUtils.java @@ -17,9 +17,9 @@ package org.apache.streampark.common.util; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.InternalConfigHolder; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.option.YarnOptions; import org.apache.streampark.shaded.org.slf4j.Logger; @@ -56,25 +56,17 @@ public final class YarnUtils { private static String rmHttpURL; - public static final String PROXY_YARN_URL = - InternalConfigHolder.get(CommonConfig.STREAMPARK_PROXY_YARN_URL()); - - public static final boolean HAS_YARN_HTTP_KERBEROS_AUTH = - "kerberos" - .equalsIgnoreCase(InternalConfigHolder.get(CommonConfig.STREAMPARK_YARN_AUTH())); - - public static final boolean HAS_YARN_HTTP_SIMPLE_AUTH = - "simple".equalsIgnoreCase(InternalConfigHolder.get(CommonConfig.STREAMPARK_YARN_AUTH())); - private YarnUtils() { } public static boolean hasYarnHttpKerberosAuth() { - return HAS_YARN_HTTP_KERBEROS_AUTH; + return "kerberos" + .equalsIgnoreCase(GlobalConfiguration.current().get(YarnOptions.HTTP_AUTHENTICATION)); } public static boolean hasYarnHttpSimpleAuth() { - return HAS_YARN_HTTP_SIMPLE_AUTH; + return "simple" + .equalsIgnoreCase(GlobalConfiguration.current().get(YarnOptions.HTTP_AUTHENTICATION)); } public static List getAppId(String appName) { @@ -127,8 +119,9 @@ public static boolean isContains(String appName) { } public static String getRMWebAppProxyURL() { - if (StringUtils.isNotBlank(PROXY_YARN_URL)) { - return PROXY_YARN_URL; + String proxyUrl = GlobalConfiguration.current().get(YarnOptions.PROXY_URL); + if (StringUtils.isNotBlank(proxyUrl)) { + return proxyUrl; } return getRMWebAppURL(); } @@ -261,7 +254,7 @@ public static String restRequest(String url, Timeout timeout) throws IOException try { return request(url, timeout); } catch (Exception e) { - if (HAS_YARN_HTTP_KERBEROS_AUTH) { + if (hasYarnHttpKerberosAuth()) { throw new IOException("yarnUtils authRestRequest error, url: " + url + ", detail: " + e, e); } throw new IOException("yarnUtils restRequest error, url: " + url + ", detail: " + e, e); @@ -285,16 +278,16 @@ public static String restRequest(String url, Timeout timeout) throws IOException private static String request(String reqUrl, Timeout timeout) throws Exception { RequestConfig config = RequestConfig.custom().setConnectTimeout(timeout).build(); - if (HAS_YARN_HTTP_KERBEROS_AUTH) { + if (hasYarnHttpKerberosAuth()) { return HadoopUtils.getUgi() .doAs( (PrivilegedExceptionAction) () -> HttpClientUtils.httpAuthGetRequest(reqUrl, config)); } String url; - if (!HAS_YARN_HTTP_SIMPLE_AUTH) { + if (!hasYarnHttpSimpleAuth()) { url = reqUrl; } else { - url = reqUrl + "?user.name=" + HadoopConfigUtils.HADOOP_USER_NAME; + url = reqUrl + "?user.name=" + HadoopConfigUtils.hadoopUserName(); } return HttpClientUtils.httpGetRequest(url, config); } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/ZooKeeperUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/ZooKeeperUtils.java deleted file mode 100644 index a53d3f4af3..0000000000 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/ZooKeeperUtils.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -import org.apache.streampark.shaded.org.slf4j.Logger; - -import org.apache.curator.RetryPolicy; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.RetryNTimes; -import org.apache.zookeeper.CreateMode; - -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -public final class ZooKeeperUtils { - - private static final Logger LOG = - StreamParkLoggerFactory.loggerFactory().getLogger(ZooKeeperUtils.class.getName()); - - private static final String CONNECT = "localhost:2181"; - private static final Map CLIENT_MAP = new ConcurrentHashMap<>(); - - private ZooKeeperUtils() { - } - - public static CuratorFramework getClient() { - return getClient(CONNECT); - } - - public static CuratorFramework getClient(String url) { - CuratorFramework existing = CLIENT_MAP.get(url); - if (existing != null) { - return existing; - } - try { - RetryPolicy retryPolicy = new RetryNTimes(5, 2000); - CuratorFramework client = - CuratorFrameworkFactory.builder() - .connectString(url) - .retryPolicy(retryPolicy) - .connectionTimeoutMs(2000) - .build(); - client.start(); - CLIENT_MAP.put(url, client); - return client; - } catch (Exception e) { - throw new IllegalStateException(e.getMessage(), e); - } - } - - public static void close(String url) { - CuratorFramework client = getClient(url); - if (client != null) { - client.close(); - CLIENT_MAP.remove(url); - } - } - - public static List listChildren(String path) throws Exception { - return listChildren(path, CONNECT); - } - - public static List listChildren(String path, String url) throws Exception { - CuratorFramework client = getClient(url); - if (client.checkExists().forPath(path) == null) { - return new ArrayList<>(); - } - return client.getChildren().forPath(path); - } - - public static boolean create(String path) throws Exception { - return create(path, null, CONNECT, false); - } - - public static boolean create(String path, String value, String url, boolean persistent) throws Exception { - try { - CuratorFramework client = getClient(url); - if (client.checkExists().forPath(path) == null) { - byte[] data = - value == null || value.isEmpty() - ? new byte[0] - : value.getBytes(StandardCharsets.UTF_8); - CreateMode mode = persistent ? CreateMode.PERSISTENT : CreateMode.EPHEMERAL; - String opResult = - client.create().creatingParentsIfNeeded().withMode(mode).forPath(path, data); - return path.equals(opResult); - } - return false; - } catch (Exception e) { - LOG.warn("Failed to create ZooKeeper path {}", path, e); - return false; - } - } - - public static boolean update(String path, String value, String url, boolean persistent) throws Exception { - try { - CuratorFramework client = getClient(url); - if (client.checkExists().forPath(path) == null) { - CreateMode mode = persistent ? CreateMode.PERSISTENT : CreateMode.EPHEMERAL; - String opResult = - client.create() - .creatingParentsIfNeeded() - .withMode(mode) - .forPath(path, value.getBytes(StandardCharsets.UTF_8)); - return path.equals(opResult); - } - client.setData().forPath(path, value.getBytes(StandardCharsets.UTF_8)); - return true; - } catch (Exception e) { - LOG.warn("Failed to update ZooKeeper path {}", path, e); - return false; - } - } - - public static void delete(String path) throws Exception { - delete(path, CONNECT); - } - - public static void delete(String path, String url) throws Exception { - try { - CuratorFramework client = getClient(url); - if (client.checkExists().forPath(path) != null) { - client.delete().deletingChildrenIfNeeded().forPath(path); - } - } catch (Exception e) { - LOG.warn("Failed to delete ZooKeeper path {}", path, e); - } - } - - public static String get(String path) throws Exception { - return get(path, CONNECT); - } - - public static String get(String path, String url) throws Exception { - try { - CuratorFramework client = getClient(url); - if (client.checkExists().forPath(path) == null) { - return null; - } - return new String(client.getData().forPath(path)); - } catch (Exception e) { - LOG.warn("Failed to read ZooKeeper path {}", path, e); - return null; - } - } -} diff --git a/streampark-common/src/test/java/org/apache/streampark/common/configuration/CommandLineParserTest.java b/streampark-common/src/test/java/org/apache/streampark/common/configuration/CommandLineParserTest.java new file mode 100644 index 0000000000..8fe2a6bc6e --- /dev/null +++ b/streampark-common/src/test/java/org/apache/streampark/common/configuration/CommandLineParserTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class CommandLineParserTest { + + @Test + void parseSeparatedInlineAndFlagValues() { + Configuration configuration = + CommandLineParser.parse( + new String[]{"--name", "orders", "--parallelism=4", "--detached"}); + + assertThat(configuration.toMap()) + .containsEntry("name", "orders") + .containsEntry("parallelism", "4") + .containsEntry("detached", "true"); + } + + @Test + void rejectPositionalAndDuplicateArgs() { + assertThatThrownBy(() -> CommandLineParser.parse(new String[]{"orders"})) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("Expected a --key"); + assertThatThrownBy( + () -> CommandLineParser.parse(new String[]{"--name", "one", "--name", "two"})) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("Duplicate command-line option"); + } + + @Test + void tokenizeQuotesAndEscapesSafely() { + assertThat(CommandLineTokenizer.tokenize("--name 'order service' --sql select\\ 1 $HOME")) + .containsExactly("--name", "order service", "--sql", "select 1", "$HOME"); + assertThatThrownBy(() -> CommandLineTokenizer.tokenize("--name 'unterminated")) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("unterminated"); + } + + @Test + void requireExplicitJvmAssignments() { + Configuration configuration = + JvmOptionsParser.parse("-Dparallelism.default=4 -D pipeline.name='orders job'"); + + assertThat(configuration.toMap()) + .containsEntry("parallelism.default", "4") + .containsEntry("pipeline.name", "orders job"); + assertThatThrownBy(() -> JvmOptionsParser.parse("-Ddetached")) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("key=value"); + } + + @Test + void buildImmutableGroupedOptions() { + Map> grouped = + GroupedOptionsParser.parse( + new String[]{"--kafka", "topic=orders", "--jdbc", "url=jdbc:h2:mem:test"}); + + assertThat(grouped.get("kafka")).containsEntry("topic", "orders"); + assertThat(grouped.get("jdbc")).containsEntry("url", "jdbc:h2:mem:test"); + assertThatThrownBy(() -> grouped.put("new", Map.of())) + .isInstanceOf(UnsupportedOperationException.class); + } +} diff --git a/streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationParserTest.java b/streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationParserTest.java new file mode 100644 index 0000000000..fbeb8f5eca --- /dev/null +++ b/streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationParserTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ConfigurationParserTest { + + @Test + void flattenYamlAndPreserveLists() { + Configuration configuration = + ConfigurationParser.parse( + "server:\n port: 8080\n hosts: [host-a, host-b]\nempty: null\n", + ConfigurationFormat.YAML, + "test YAML"); + + assertThat(configuration.toMap()) + .containsEntry("server.port", "8080") + .containsEntry("server.hosts", "host-a,host-b") + .doesNotContainKey("empty"); + } + + @Test + void rejectDuplicateAndFlattenedYamlKeys() { + assertThatThrownBy( + () -> ConfigurationParser.parse( + "server:\n port: 8080\nserver.port: 9090\n", + ConfigurationFormat.YAML, + "ambiguous YAML")) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("Duplicate configuration key"); + } + + @Test + void hoconShouldResolveSubstitutions() { + Configuration configuration = + ConfigurationParser.parse( + "base.port = 8080\nserver.port = ${base.port}\n", + ConfigurationFormat.HOCON, + "test HOCON"); + + assertThat(configuration.getString("server.port")).isEqualTo("8080"); + } + + @Test + void propertiesShouldPreserveExactKeys() { + Configuration configuration = + ConfigurationParser.parse( + "server.port=8080\nfeature.enabled=true\n", + ConfigurationFormat.PROPERTIES, + "test properties"); + + assertThat(configuration.toMap()) + .containsEntry("server.port", "8080") + .containsEntry("feature.enabled", "true"); + } + + @Test + void fileFormatShouldBeInferredFromExtension(@TempDir Path temporaryDirectory) throws IOException { + Path file = temporaryDirectory.resolve("application.yaml"); + Files.writeString(file, "server:\n port: 8080\n"); + + Configuration configuration = ConfigurationParser.parse(file); + + assertThat(configuration.getString("server.port")).isEqualTo("8080"); + assertThat(configuration.origin("server.port").orElseThrow().name()) + .isEqualTo(file.toAbsolutePath().normalize().toString()); + } +} diff --git a/streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationTest.java b/streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationTest.java new file mode 100644 index 0000000000..8070c7b2bb --- /dev/null +++ b/streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationTest.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class ConfigurationTest { + + private static final ConfigOption PORT = + ConfigOptions.key("server.port") + .intType() + .defaultValue(8080) + .check(value -> value > 0, "port must be positive") + .withDescription("Server port.") + .build(); + + @Test + void preferHigherPrioritySource() { + Configuration configuration = + Configuration.builder() + .add("command line", ConfigSource.COMMAND_LINE, Map.of(PORT.key(), "9090")) + .add("file", ConfigSource.FILE, Map.of(PORT.key(), "8081")) + .build(); + + assertThat(configuration.get(PORT)).isEqualTo(9090); + assertThat(configuration.origin(PORT.key())) + .contains(ConfigOrigin.of(ConfigSource.COMMAND_LINE, "command line")); + } + + @Test + void laterValueWithinSameSourceShouldWin() { + Configuration configuration = + Configuration.builder() + .add("base file", ConfigSource.FILE, Map.of(PORT.key(), "8081")) + .add("override file", ConfigSource.FILE, Map.of(PORT.key(), "8082")) + .build(); + + assertThat(configuration.get(PORT)).isEqualTo(8082); + } + + @Test + void convertStructuredAndTextValues() { + ConfigOption timeout = + ConfigOptions.key("request.timeout") + .durationType(ChronoUnit.SECONDS) + .noDefaultValue() + .withDescription("Request timeout.") + .build(); + ConfigOption size = + ConfigOptions.key("buffer.size") + .dataSizeType() + .noDefaultValue() + .withDescription("Buffer size.") + .build(); + ConfigOption> hosts = + ConfigOptions.key("hosts") + .stringListType() + .noDefaultValue() + .withDescription("Target hosts.") + .build(); + Configuration configuration = + Configuration.builder() + .add( + "test", + ConfigSource.FILE, + Map.of( + timeout.key(), "15", + size.key(), "2 MiB", + hosts.key(), Arrays.asList("host-a", "host-b"))) + .build(); + + assertThat(configuration.get(timeout)).isEqualTo(Duration.ofSeconds(15)); + assertThat(configuration.get(size)).isEqualTo(DataSize.ofMebiBytes(2)); + assertThat(configuration.get(hosts)).containsExactly("host-a", "host-b"); + } + + @Test + void preferCanonicalKey() { + ConfigOption option = + ConfigOptions.key("current.key") + .stringType() + .noDefaultValue() + .withFallbackKeys("legacy.key") + .withDescription("Migrated option.") + .build(); + Configuration fallbackOnly = + Configuration.builder() + .add("test", ConfigSource.FILE, Map.of("legacy.key", "legacy")) + .build(); + Configuration both = + Configuration.builder(fallbackOnly) + .add("test", ConfigSource.FILE, Map.of("current.key", "current")) + .build(); + + assertThat(fallbackOnly.get(option)).isEqualTo("legacy"); + assertThat(both.get(option)).isEqualTo("current"); + } + + @Test + void invalidSensitiveValueShouldBeRedacted() { + ConfigOption secret = + ConfigOptions.key("service.secret") + .intType() + .noDefaultValue() + .sensitive() + .withDescription("Sensitive numeric value.") + .build(); + Configuration configuration = + Configuration.builder() + .add("test", ConfigSource.FILE, Map.of(secret.key(), "clear-text-value")) + .build(); + + assertThatThrownBy(() -> configuration.get(secret)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining(SensitiveKeys.MASK) + .hasMessageNotContaining("clear-text-value"); + assertThat(configuration.toRedactedMap()).containsEntry(secret.key(), SensitiveKeys.MASK); + } + + @Test + void redactFlinkSecrets() { + assertThat( + Arrays.asList( + "fs.azure.account.key.storage", + "security.apikey", + "security.auth-params", + "security.basic-auth", + "security.jaas.config", + "security.http-headers")) + .allMatch(SensitiveKeys::isSensitive); + } + + @Test + void snapshotsAndViewsShouldBeImmutable() { + Configuration configuration = + Configuration.builder() + .add("test", ConfigSource.FILE, Map.of("app.name", "orders")) + .build(); + + assertThat(configuration.subset("app.").toMap()).containsExactly(Map.entry("name", "orders")); + assertThatThrownBy(() -> configuration.toMap().put("new", "value")) + .isInstanceOf(UnsupportedOperationException.class); + assertThat(Configuration.empty().toMap()).isEqualTo(Collections.emptyMap()); + } + + @Test + void copyMutableValues() { + ConfigOption> hosts = + ConfigOptions.key("hosts") + .stringListType() + .noDefaultValue() + .withDescription("Target hosts.") + .build(); + List source = new ArrayList<>(Collections.singletonList("host-a")); + Configuration configuration = + Configuration.builder() + .add("test", ConfigSource.FILE, Map.of(hosts.key(), source)) + .build(); + + source.add("host-b"); + + assertThat(configuration.get(hosts)).containsExactly("host-a"); + assertThatThrownBy(() -> configuration.get(hosts).add("host-c")) + .isInstanceOf(UnsupportedOperationException.class); + + List defaultHosts = new ArrayList<>(Collections.singletonList("default-a")); + ConfigOption> optionWithDefault = + ConfigOptions.key("default.hosts") + .stringListType() + .defaultValue(defaultHosts) + .withDescription("Default hosts.") + .build(); + defaultHosts.add("default-b"); + + assertThat(optionWithDefault.defaultValue()).containsExactly("default-a"); + assertThatThrownBy(() -> optionWithDefault.defaultValue().add("default-c")) + .isInstanceOf(UnsupportedOperationException.class); + + List pending = new ArrayList<>(Collections.singletonList("pending-a")); + ConfigurationLoader loader = + new ConfigurationLoader() + .add(ConfigSource.FILE, "test", Map.of(hosts.key(), pending)); + pending.add("pending-b"); + + assertThat(loader.load().get(hosts)).containsExactly("pending-a"); + } + + @Test + void rejectInvalidAndMissingValues() { + Configuration invalid = + Configuration.builder() + .add("application.yaml", ConfigSource.FILE, Map.of(PORT.key(), "zero")) + .build(); + + assertThatThrownBy(() -> invalid.get(PORT)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining(PORT.key()) + .hasMessageContaining("application.yaml"); + assertThatThrownBy( + () -> Configuration.empty() + .get( + ConfigOptions.key("required") + .stringType() + .noDefaultValue() + .withDescription("Required value.") + .build())) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("required"); + } + + @Test + void rejectNegativeDurations() { + ConfigOption timeout = + ConfigOptions.key("request.timeout") + .durationType() + .noDefaultValue() + .withDescription("Request timeout.") + .build(); + Configuration durationValue = + Configuration.builder() + .add("test", ConfigSource.FILE, Map.of(timeout.key(), Duration.ofSeconds(-1))) + .build(); + Configuration isoValue = + Configuration.builder() + .add("test", ConfigSource.FILE, Map.of(timeout.key(), "-PT1S")) + .build(); + + assertThatThrownBy(() -> durationValue.get(timeout)) + .isInstanceOf(ConfigException.class) + .hasRootCauseMessage("duration must not be negative"); + assertThatThrownBy(() -> isoValue.get(timeout)) + .isInstanceOf(ConfigException.class) + .hasRootCauseMessage("duration must not be negative"); + } +} diff --git a/streampark-common/src/test/java/org/apache/streampark/common/configuration/WorkspaceTest.java b/streampark-common/src/test/java/org/apache/streampark/common/configuration/WorkspaceTest.java new file mode 100644 index 0000000000..80ead09744 --- /dev/null +++ b/streampark-common/src/test/java/org/apache/streampark/common/configuration/WorkspaceTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.configuration; + +import org.apache.streampark.common.configuration.option.WorkspaceOptions; +import org.apache.streampark.common.enums.StorageType; +import org.apache.streampark.common.util.HadoopUtils; + +import org.apache.hadoop.fs.FileSystem; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class WorkspaceTest { + + @Test + void bindWorkspaceToSnapshot() { + Configuration first = + Configuration.builder() + .set(WorkspaceOptions.LOCAL_ROOT, "/tmp/first/", "test") + .build(); + Configuration second = + Configuration.builder() + .set(WorkspaceOptions.LOCAL_ROOT, "/tmp/second", "test") + .build(); + + Workspace firstWorkspace = Workspace.of(StorageType.LFS, first); + Workspace secondWorkspace = Workspace.of(StorageType.LFS, second); + + assertThat(firstWorkspace.root).isEqualTo("/tmp/first"); + assertThat(firstWorkspace.workspace).isEqualTo("/tmp/first/workspace"); + assertThat(firstWorkspace.jars).isEqualTo("/tmp/first/jars"); + assertThat(secondWorkspace.root).isEqualTo("/tmp/second"); + } + + @Test + void avoidDuplicateRootSeparators() { + Configuration configuration = + Configuration.builder() + .set(WorkspaceOptions.LOCAL_ROOT, "/", "test") + .build(); + + Workspace workspace = Workspace.of(StorageType.LFS, configuration); + + assertThat(workspace.root).isEqualTo("/"); + assertThat(workspace.workspace).isEqualTo("/workspace"); + assertThat(workspace.plugins).isEqualTo("/plugins"); + } + + @Test + void rejectUnsupportedRemoteScheme() { + Configuration configuration = + Configuration.builder() + .set(WorkspaceOptions.REMOTE_ROOT, "s3://bucket/streampark", "test") + .build(); + + assertThatThrownBy(() -> Workspace.of(StorageType.HDFS, configuration)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining(WorkspaceOptions.REMOTE_ROOT.key()) + .hasMessageContaining("hdfs scheme"); + } + + @Test + void reportMalformedRemoteRoot() { + Configuration configuration = + Configuration.builder() + .set(WorkspaceOptions.REMOTE_ROOT, "/invalid path", "test") + .build(); + + assertThatThrownBy(() -> Workspace.of(StorageType.HDFS, configuration)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining(WorkspaceOptions.REMOTE_ROOT.key()) + .hasMessageContaining("valid URI"); + } + + @Test + void useDefaultHdfsAuthority() { + org.apache.hadoop.conf.Configuration hadoopConfiguration = HadoopUtils.hadoopConf(); + String previousDefault = hadoopConfiguration.get(FileSystem.FS_DEFAULT_NAME_KEY); + hadoopConfiguration.set(FileSystem.FS_DEFAULT_NAME_KEY, "hdfs://primary:8020"); + try { + Configuration authorityless = + Configuration.builder() + .set(WorkspaceOptions.REMOTE_ROOT, "hdfs:///streampark/", "test") + .build(); + Configuration anotherFileSystem = + Configuration.builder() + .set( + WorkspaceOptions.REMOTE_ROOT, + "hdfs://secondary:8020/streampark", + "test") + .build(); + + assertThat(Workspace.of(StorageType.HDFS, authorityless).root) + .isEqualTo("hdfs://primary:8020/streampark"); + assertThatThrownBy(() -> Workspace.of(StorageType.HDFS, anotherFileSystem)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("does not match fs.defaultFS"); + } finally { + if (previousDefault == null) { + hadoopConfiguration.unset(FileSystem.FS_DEFAULT_NAME_KEY); + } else { + hadoopConfiguration.set(FileSystem.FS_DEFAULT_NAME_KEY, previousDefault); + } + } + } + + @Test + void rejectEarlyWorkspaceSnapshot() { + Configuration changed = + Configuration.builder(GlobalConfiguration.current()) + .set(WorkspaceOptions.LOCAL_ROOT, "/different-bootstrap-root", "test") + .build(); + + assertThatThrownBy(() -> Workspace.verifyInitializedFrom(changed)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("before configuration bootstrap completed"); + } +} diff --git a/streampark-common/src/test/java/org/apache/streampark/common/core/FlinkVersionTest.java b/streampark-common/src/test/java/org/apache/streampark/common/core/FlinkVersionTest.java new file mode 100644 index 0000000000..62a4f8582a --- /dev/null +++ b/streampark-common/src/test/java/org/apache/streampark/common/core/FlinkVersionTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.core; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class FlinkVersionTest { + + @Test + void matchSupportedShimsRange(@TempDir Path temporaryDirectory) throws IOException { + assertThat(flinkVersion(temporaryDirectory, "1.17.2").checkVersion(false)).isFalse(); + assertThat(flinkVersion(temporaryDirectory, "1.18.0").checkVersion(false)).isTrue(); + assertThat(flinkVersion(temporaryDirectory, "1.20.1").checkVersion(false)).isTrue(); + assertThat(flinkVersion(temporaryDirectory, "2.0.0").checkVersion(false)).isTrue(); + assertThat(flinkVersion(temporaryDirectory, "2.3.0").checkVersion(false)).isTrue(); + assertThat(flinkVersion(temporaryDirectory, "2.4.0").checkVersion(false)).isFalse(); + } + + private static FlinkVersion flinkVersion(Path temporaryDirectory, String version) throws IOException { + Path flinkHome = temporaryDirectory.resolve("flink-" + version); + Path libDirectory = Files.createDirectories(flinkHome.resolve("lib")); + Files.createFile(libDirectory.resolve("flink-dist-" + version + ".jar")); + return new FlinkVersion(flinkHome.toString()); + } +} diff --git a/streampark-common/src/test/java/org/apache/streampark/common/util/DeflaterUtilsTest.java b/streampark-common/src/test/java/org/apache/streampark/common/util/DeflaterUtilsTest.java new file mode 100644 index 0000000000..b7e2a22b26 --- /dev/null +++ b/streampark-common/src/test/java/org/apache/streampark/common/util/DeflaterUtilsTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; + +class DeflaterUtilsTest { + + @Test + void shouldRoundTripUtf8Text() { + String text = "SELECT '\u6D41\u5904\u7406' AS name;"; + + assertThat(DeflaterUtils.unzipString(DeflaterUtils.zipString(text))).isEqualTo(text); + } + + @Test + void shouldPreserveWhitespaceOnlyText() { + String text = " \n\t "; + + assertThat(DeflaterUtils.unzipString(DeflaterUtils.zipString(text))).isEqualTo(text); + } + + @Test + void returnNullForInvalidInput() { + String compressed = DeflaterUtils.zipString("SELECT * FROM source;"); + String truncated = compressed.substring(0, compressed.length() - 4); + + assertThat(DeflaterUtils.unzipString("not-base64")).isNull(); + assertTimeoutPreemptively( + Duration.ofSeconds(1), () -> assertThat(DeflaterUtils.unzipString(truncated)).isNull()); + } + + @Test + void handleEmptyAndNullInput() { + assertTimeoutPreemptively( + Duration.ofSeconds(1), () -> assertThat(DeflaterUtils.unzipString("")).isEmpty()); + assertThat(DeflaterUtils.zipString(null)).isEmpty(); + assertThat(DeflaterUtils.unzipString(null)).isNull(); + } + + @Test + void shouldNormalizeLegacyNestedCompression() { + String sql = "CREATE TABLE sink (id BIGINT);"; + String nested = DeflaterUtils.zipString(DeflaterUtils.zipString(sql)); + + assertThat(DeflaterUtils.toPlainText(sql)).isEqualTo(sql); + assertThat(DeflaterUtils.toPlainText(nested)).isEqualTo(sql); + assertThat(DeflaterUtils.unzipString(DeflaterUtils.compressForStorage(nested))).isEqualTo(sql); + } +} diff --git a/streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java b/streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java new file mode 100644 index 0000000000..050670566b --- /dev/null +++ b/streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.streampark.common.configuration.ConfigException; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies version-aware loading of Flink's legacy and standard YAML formats. */ +class FlinkConfigurationUtilsTest { + + @TempDir + Path temporaryDirectory; + + @Test + void loadNestedYamlAndPreserveLists() throws Exception { + Path configFile = temporaryDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME); + Files.writeString( + configFile, + "env:\n" + + " java:\n" + + " opts:\n" + + " all: --add-opens=java.base/java.util=ALL-UNNAMED\n" + + "high-availability:\n" + + " namespaces: [primary, secondary]\n"); + + Map configuration = + FlinkConfigurationUtils.loadConfigurationFromFile(configFile.toFile()); + + assertThat(configuration.get("env.java.opts.all")) + .isEqualTo("--add-opens=java.base/java.util=ALL-UNNAMED"); + assertThat(configuration.get("high-availability.namespaces")) + .isEqualTo("[primary, secondary]"); + } + + @Test + void selectOnlyLegacyYamlBeforeFlink119() throws Exception { + Files.writeString( + temporaryDirectory.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME), + "pipeline.name: legacy\n"); + Files.writeString( + temporaryDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME), + "pipeline:\n name: standard\n"); + + Map configuration = + FlinkConfigurationUtils.loadConfiguration( + temporaryDirectory.toString(), "1.18.2"); + + assertThat(FlinkConfigurationUtils.usesStandardYaml( + temporaryDirectory.toString(), "1.18.2")).isFalse(); + assertThat(configuration.get("pipeline.name")).isEqualTo("legacy"); + } + + @Test + void selectFlink119YamlByPrecedence() throws Exception { + Path legacyFile = + temporaryDirectory.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME); + Files.writeString(legacyFile, "pipeline.name: legacy\n"); + Files.writeString( + temporaryDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME), + "pipeline:\n name: standard\n"); + + assertThat(FlinkConfigurationUtils.usesStandardYaml( + temporaryDirectory.toString(), "1.19.0")).isFalse(); + assertThat( + FlinkConfigurationUtils.loadConfiguration( + temporaryDirectory.toString(), "1.19.0") + .get("pipeline.name")) + .isEqualTo("legacy"); + + Files.delete(legacyFile); + + assertThat(FlinkConfigurationUtils.usesStandardYaml( + temporaryDirectory.toString(), "1.19.1")).isTrue(); + assertThat( + FlinkConfigurationUtils.loadConfiguration( + temporaryDirectory.toString(), "1.19.1") + .get("pipeline.name")) + .isEqualTo("standard"); + } + + @Test + void supportLegacyThroughFlink120() throws Exception { + Path legacyFile = + temporaryDirectory.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME); + Files.writeString(legacyFile, "pipeline.name: legacy\n"); + Files.writeString( + temporaryDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME), + "pipeline:\n name: standard\n"); + + assertThat(FlinkConfigurationUtils.usesStandardYaml( + temporaryDirectory.toString(), "1.20.0")).isFalse(); + assertThat( + FlinkConfigurationUtils.loadConfiguration( + temporaryDirectory.toString(), "1.20.0") + .get("pipeline.name")) + .isEqualTo("legacy"); + + Files.delete(legacyFile); + + assertThat(FlinkConfigurationUtils.usesStandardYaml( + temporaryDirectory.toString(), "1.20.3")).isTrue(); + assertThat( + FlinkConfigurationUtils.loadConfiguration( + temporaryDirectory.toString(), "1.20.3") + .get("pipeline.name")) + .isEqualTo("standard"); + } + + @Test + void requireConfigYamlFromFlink2() throws Exception { + Files.writeString( + temporaryDirectory.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME), + "pipeline.name: legacy\n"); + + assertThatThrownBy( + () -> FlinkConfigurationUtils.loadConfiguration( + temporaryDirectory.toString(), "2.0.0")) + .isInstanceOf(ConfigException.class) + .hasMessageContaining(FlinkConfigurationUtils.FLINK_CONF_FILENAME); + + Files.writeString( + temporaryDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME), + "pipeline:\n name: standard\n"); + + assertThat(FlinkConfigurationUtils.usesStandardYaml( + temporaryDirectory.toString(), "2.0.0")).isTrue(); + assertThat( + FlinkConfigurationUtils.loadConfiguration( + temporaryDirectory.toString(), "2.0.0") + .get("pipeline.name")) + .isEqualTo("standard"); + } + + @Test + void readLegacyFileAsUtf8() throws Exception { + Path file = + temporaryDirectory.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME); + Files.writeString(file, "pipeline.name: \u4e2d\u6587\u4f5c\u4e1a\n", StandardCharsets.UTF_8); + + Map configuration = + FlinkConfigurationUtils.loadConfigurationFromFile(file.toFile()); + + assertThat(configuration.get("pipeline.name")).isEqualTo("\u4e2d\u6587\u4f5c\u4e1a"); + } + + @Test + void rejectMissingConfigurationDirectory() { + assertThatThrownBy( + () -> FlinkConfigurationUtils.loadConfiguration( + temporaryDirectory.resolve("missing").toString(), "1.20.0")) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("does not exist"); + } +} diff --git a/streampark-common/src/test/java/org/apache/streampark/common/util/PropertiesUtilsTestCase.java b/streampark-common/src/test/java/org/apache/streampark/common/util/PropertiesUtilsTestCase.java deleted file mode 100644 index 16dfa79702..0000000000 --- a/streampark-common/src/test/java/org/apache/streampark/common/util/PropertiesUtilsTestCase.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.common.util; - -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class PropertiesUtilsTestCase { - - @Test - void testExtractProgramArgs() { - String args = - "mysql-sync-database " - + "--database employees " - + "--mysql-conf hostname=127.0.0.1 " - + "--mysql-conf port=3306 " - + "--mysql-conf username=root " - + "--mysql-conf password=123456 " - + "--mysql-conf database-name=employees " - + "--including-tables 'test|test.*' " - + "--excluding-tables \"emp_*\" " - + "--query 'select * from employees where age > 20' " - + "--sink-conf fenodes=127.0.0.1:8030 " - + "--sink-conf username=root " - + "--sink-conf password= " - + "--sink-conf jdbc-url=jdbc:mysql://127.0.0.1:9030 " - + "--sink-conf sink.label-prefix=label" - + "--table-conf replication_num=1"; - List programArgs = FlinkConfigurationUtils.extractArguments(args); - assertTrue(programArgs.contains("username=root")); - } - - @Test - void testDynamicProperties() { - String dynamicProperties = - "\n" - + "-Denv.java.opts1=\"-Dfile.encoding=UTF-8\"\n" - + "-Denv.java.opts2 = \"-Dfile.enc\\\"oding=UTF-8\"\n" - + "-Denv.java.opts3 = \" -Dfile.encoding=UTF-8\"\n" - + "-Dyarn.application.id=123\n" - + "-Dyarn.application.name=\"streampark job\"\n" - + "-Dyarn.application.queue=flink\n" - + "-Ddiy.param.name=apache streampark\n" - + "\n"; - - java.util.Map map = - FlinkConfigurationUtils.extractDynamicProperties(dynamicProperties); - assertEquals("-Dfile.encoding=UTF-8", map.get("env.java.opts1")); - assertEquals("-Dfile.enc\\\"oding=UTF-8", map.get("env.java.opts2")); - assertEquals(" -Dfile.encoding=UTF-8", map.get("env.java.opts3")); - assertEquals("123", map.get("yarn.application.id")); - assertEquals("streampark job", map.get("yarn.application.name")); - assertEquals("flink", map.get("yarn.application.queue")); - assertEquals("apache streampark", map.get("diy.param.name")); - } -} diff --git a/streampark-common/src/test/java/org/apache/streampark/common/util/YamlParserTest.java b/streampark-common/src/test/java/org/apache/streampark/common/util/YamlParserTest.java new file mode 100644 index 0000000000..da3a73023b --- /dev/null +++ b/streampark-common/src/test/java/org/apache/streampark/common/util/YamlParserTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.apache.streampark.common.configuration.ConfigException; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies strict YAML parsing and the bounded compatibility fallback. */ +class YamlParserTest { + + @Test + void parseEditorMultilineScalar() { + String yaml = + "source:\n" + + " tables:\n" + + " db.table_a,\n" + + " # db.table_b,\n" + + " db.table_c\n" + + "transforms:\n" + + " - source-table: db.table_a\n" + + " projection: |\n" + + " id,\n" + + " name\n"; + + Map document = YamlParser.parse(yaml); + + @SuppressWarnings("unchecked") + Map source = (Map) document.get("source"); + assertThat(source) + .containsEntry("tables", "db.table_a, db.table_c"); + assertThat(document.get("transforms")).isInstanceOf(java.util.List.class); + } + + @Test + void quoteUnsafeScalarsOnRetry() { + Map document = + YamlParser.parse( + "security:\n" + + " keytab: !unquoted-keytab\n" + + "state:\n" + + " directory: >hdfs:///checkpoints\n"); + + assertThat(YamlParser.flatten(document)) + .containsEntry("security.keytab", "!unquoted-keytab") + .containsEntry("state.directory", ">hdfs:///checkpoints"); + } + + @Test + void decodeUtf16LittleEndianBom() { + byte[] content = "pipeline:\n name: demo\n".getBytes(StandardCharsets.UTF_16LE); + byte[] bytes = new byte[content.length + 2]; + bytes[0] = (byte) 0xFF; + bytes[1] = (byte) 0xFE; + System.arraycopy(content, 0, bytes, 2, content.length); + + Map document = + YamlParser.parse(new ByteArrayInputStream(bytes)); + + assertThat(YamlParser.flatten(document)).containsEntry("pipeline.name", "demo"); + } + + @Test + void serializeListsUsingYamlFlowSyntax() { + assertThat(YamlParser.toFlowString(Arrays.asList("host-a", "host-b"))) + .isEqualTo("[host-a, host-b]"); + } + + @Test + void rejectDuplicateKeysSafely() { + assertThatThrownBy(() -> YamlParser.parse("password: first\npassword: second\n")) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("duplicate key password") + .hasMessageNotContaining("first") + .hasMessageNotContaining("second"); + } +} diff --git a/streampark-console/streampark-console-service/pom.xml b/streampark-console/streampark-console-service/pom.xml index dddc5ebaf3..479eeadd96 100644 --- a/streampark-console/streampark-console-service/pom.xml +++ b/streampark-console/streampark-console-service/pom.xml @@ -35,7 +35,7 @@ UTF-8 42.5.1 3.5.3.1 - 1.17 + 1.18 streampark-console-webapp 512m @@ -366,25 +366,19 @@ streampark-common + - org.apache.streampark - streampark-flink-shims-base - ${project.version} + com.squareup.okhttp3 + okhttp org.apache.streampark - streampark-flink-shims-base-v2 + streampark-flink-shims-base ${project.version} - - org.apache.streampark - streampark-flink-shims_flink-1.17 - ${project.version} - provided - org.apache.streampark streampark-flink-shims_flink-1.18 @@ -600,13 +594,6 @@ false false - - - org.apache.streampark - streampark-flink-shims_flink-1.17 - ${project.version} - ${project.build.directory}/shims - org.apache.streampark diff --git a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh index fce57d3ed6..3a1a765435 100755 --- a/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh +++ b/streampark-console/streampark-console-service/src/main/assembly/bin/streampark.sh @@ -286,7 +286,7 @@ print_logo() { printf ' %s ___/ / /_/ / / __/ /_/ / / / / / / /_/ / /_/ / / / ,< %s\n' $PRIMARY $RESET printf ' %s /____/\__/_/ \___/\__,_/_/ /_/ /_/ ____/\__,_/_/ /_/|_| %s\n' $PRIMARY $RESET printf ' %s /_/ %s\n\n' $PRIMARY $RESET - printf ' %s Version: 3.0.0-SNAPSHOT %s\n' $BLUE $RESET + printf ' %s Version: 3.0.0 %s\n' $BLUE $RESET printf ' %s WebSite: https://streampark.apache.org%s\n' $BLUE $RESET printf ' %s GitHub : http://github.com/apache/streampark%s\n\n' $BLUE $RESET printf ' %s ──────── Apache StreamPark, Make stream processing easier ô~ô!%s\n\n' $PRIMARY $RESET diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/ConsoleOptions.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/ConsoleOptions.java new file mode 100644 index 0000000000..b40456a22d --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/ConsoleOptions.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.base.config; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; +import org.apache.streampark.common.configuration.DataSize; + +/** Configuration options owned by the StreamPark console service. */ +public final class ConsoleOptions { + + public static final ConfigOption BUILD_LOG_READ_MAX_SIZE = + ConfigOptions.key("streampark.read-log.max-size") + .dataSizeType() + .defaultValue(DataSize.ofMebiBytes(1)) + .check(value -> value.bytes() > 0, "read size must be greater than zero") + .withDescription("Maximum number of build-log bytes returned by one read operation.") + .build(); + + public static final ConfigOption DATABASE_DIALECT = + ConfigOptions.key("spring.profiles.active") + .stringType() + .defaultValue("h2") + .withDescription("Active database profile used by the console service.") + .build(); + + private ConsoleOptions() { + } +} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringConfigurationInitializer.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringConfigurationInitializer.java new file mode 100644 index 0000000000..b60c108333 --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringConfigurationInitializer.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.base.config; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigSource; +import org.apache.streampark.common.configuration.Configuration; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.Workspace; +import org.apache.streampark.common.configuration.option.CoreOptions; +import org.apache.streampark.common.configuration.option.DockerOptions; +import org.apache.streampark.common.configuration.option.HadoopOptions; +import org.apache.streampark.common.configuration.option.MavenOptions; +import org.apache.streampark.common.configuration.option.WorkspaceOptions; +import org.apache.streampark.common.configuration.option.YarnOptions; +import org.apache.streampark.common.util.SystemPropertyUtils; +import org.apache.streampark.flink.kubernetes.configuration.KubernetesOptions; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Bridges Spring's resolved environment into StreamPark's typed server configuration. + * + *

Option declarations remain with the modules that consume them. This Console-side composition + * root owns only the list of options whose values may be overridden by Spring property sources, + * such as command-line arguments, environment variables, and profile-specific configuration. + * Keeping that list here prevents framework integration concerns from leaking into common or + * engine-specific option catalogs. + */ +@Slf4j +@Component +public final class SpringConfigurationInitializer { + + private static final String SPRING_ENVIRONMENT_ORIGIN = "resolved Spring environment"; + + /** + * Options read by server components after the Spring context has completed property resolution. + * + *

This is intentionally an explicit integration boundary. Adding an option to its owning + * catalog does not implicitly expose it to Spring or create a global registration side effect. + */ + private static final List> SPRING_BOUND_OPTIONS = List.of( + CoreOptions.APP_HOME, + WorkspaceOptions.LOCAL_ROOT, + WorkspaceOptions.REMOTE_ROOT, + HadoopOptions.USER_NAME, + HadoopOptions.KERBEROS_DEBUG, + HadoopOptions.KERBEROS_ENABLED, + HadoopOptions.KERBEROS_PRINCIPAL, + HadoopOptions.KERBEROS_KEYTAB, + HadoopOptions.KERBEROS_KRB5, + HadoopOptions.KERBEROS_TICKET_LIFETIME, + YarnOptions.PROXY_URL, + YarnOptions.HTTP_AUTHENTICATION, + DockerOptions.HOST, + DockerOptions.MAX_CONNECTIONS, + DockerOptions.CONNECTION_TIMEOUT, + DockerOptions.RESPONSE_TIMEOUT, + MavenOptions.SETTINGS_PATH, + MavenOptions.REPOSITORY_URL, + MavenOptions.USER_NAME, + MavenOptions.PASSWORD, + ConsoleOptions.BUILD_LOG_READ_MAX_SIZE, + ConsoleOptions.DATABASE_DIALECT, + KubernetesOptions.JOB_STATUS_REQUEST_TIMEOUT, + KubernetesOptions.JOB_STATUS_CACHE_TIMEOUT, + KubernetesOptions.METRIC_REQUEST_TIMEOUT, + KubernetesOptions.JOB_STATUS_POLL_INTERVAL, + KubernetesOptions.METRIC_POLL_INTERVAL, + KubernetesOptions.SILENT_STATE_TRACKING_RETENTION, + KubernetesOptions.INGRESS_CLASS); + + /** + * Publishes Spring-resolved values as one validated runtime configuration layer. + * + *

Every supplied option is type-checked before the snapshot becomes globally visible. + * Workspace verification then detects any component that accessed constant workspace paths + * before Spring resolution completed. + * + * @param environment fully prepared Spring environment + * @return the installed immutable configuration snapshot + */ + public Configuration initialize(Environment environment) { + Objects.requireNonNull(environment, "environment must not be null"); + Map resolvedValues = collect(environment); + Configuration springOverrides = Configuration.builder() + .add(SPRING_ENVIRONMENT_ORIGIN, ConfigSource.RUNTIME, resolvedValues) + .build(); + Configuration candidate = Configuration.builder(GlobalConfiguration.current()) + .add(springOverrides) + .build(); + + // Conversion is normally lazy. The process bootstrap boundary validates every supplied + // Spring override eagerly so invalid server settings fail before background services start. + SPRING_BOUND_OPTIONS.stream() + .filter(option -> resolvedValues.containsKey(option.key())) + .forEach(candidate::get); + + GlobalConfiguration.update(candidate); + initializeHadoopUser(environment, candidate); + Workspace.verifyInitializedFrom(candidate); + log.info( + "Applied {} Spring-resolved StreamPark configuration overrides: {}", + springOverrides.size(), + springOverrides.toRedactedMap()); + return candidate; + } + + private static void initializeHadoopUser( + Environment environment, + Configuration configuration) { + // Hadoop reads this legacy JVM property during its own static bootstrap. Set it before + // Workspace resolves the remote filesystem so the configured identity is effective. + String userName = environment.getProperty( + HadoopOptions.HADOOP_USER_NAME_PROPERTY, + configuration.get(HadoopOptions.USER_NAME)); + SystemPropertyUtils.set(HadoopOptions.HADOOP_USER_NAME_PROPERTY, userName); + log.info( + "Initialized Hadoop system property: {}={}", + HadoopOptions.HADOOP_USER_NAME_PROPERTY, + userName); + } + + private static Map collect(Environment environment) { + Map resolved = new LinkedHashMap<>(); + for (ConfigOption option : SPRING_BOUND_OPTIONS) { + if (environment.containsProperty(option.key())) { + String value = environment.getProperty(option.key()); + if (value != null) { + resolved.put(option.key(), value); + } + } + } + return resolved; + } +} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java index 451694395f..8894c41206 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java @@ -17,32 +17,49 @@ package org.apache.streampark.console.base.config; -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.util.PropertiesUtils; -import org.apache.streampark.common.util.SystemPropertyUtils; +import org.apache.streampark.common.configuration.ConfigSource; +import org.apache.streampark.common.configuration.Configuration; +import org.apache.streampark.common.configuration.ConfigurationLoader; +import org.apache.streampark.common.configuration.ConfigurationParser; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.option.CoreOptions; import org.apache.streampark.console.base.util.WebUtils; import org.apache.commons.lang3.StringUtils; -import com.google.common.collect.Maps; - import java.io.File; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; -public class SpringProperties { +/** + * Loads the console configuration file and derives Spring Boot properties. + * + *

The method also installs the initial StreamPark configuration snapshot. It does not copy + * application values into JVM system properties: Spring and StreamPark receive the same source + * values, while each framework retains its own explicit precedence rules. + */ +public final class SpringProperties { + /** Returns Spring Boot defaults combined with the external StreamPark configuration file. */ public static Properties get() { - // 1) get spring config Properties springConfig = getSpringConfig(); - // 2) get user config - Properties userConfig = getUserConfig(); - // 3) merge config + Configuration userConfiguration = getUserConfig(); + Properties userConfig = new Properties(); + userConfig.putAll(userConfiguration.toMap()); mergeConfig(userConfig, springConfig); - // 4) datasource dataSourceConfig(userConfig, springConfig); - // 5) system.setProperties - springConfig.forEach((k, v) -> SystemPropertyUtils.set(k.toString(), v.toString())); + + Map defaults = new LinkedHashMap<>(); + springConfig.forEach((key, value) -> defaults.put(key.toString(), value)); + Configuration configuration = new ConfigurationLoader() + .add(ConfigSource.DEFAULTS, "console defaults", defaults) + .add(ConfigSource.FILE, "conf/config.yaml", userConfiguration.toMap()) + .addSystemProperties() + .load(); + GlobalConfiguration.update(configuration); return springConfig; } @@ -110,7 +127,7 @@ private static void dataSourceConfig(Properties userConfig, Properties springCon } private static void mergeConfig(Properties userConfig, Properties springConfig) { - Map configMapping = Maps.newHashMap(); + Map configMapping = new HashMap<>(); configMapping.put("datasource.username", "spring.datasource.username"); configMapping.put("datasource.password", "spring.datasource.password"); configMapping.put("datasource.url", "spring.datasource.url"); @@ -126,7 +143,7 @@ private static void mergeConfig(Properties userConfig, Properties springConfig) }); } - private static Properties getUserConfig() { + private static Configuration getUserConfig() { String appHome = WebUtils.getAppHome(); if (StringUtils.isBlank(appHome)) { throw new ExceptionInInitializerError( @@ -134,14 +151,11 @@ private static Properties getUserConfig() { "[StreamPark] The system initialization check failed. If started local for development and debugging," + " please ensure the -D%s parameter is clearly specified," + " more detail: https://streampark.apache.org/docs/development/development/", - ConfigKeys.KEY_APP_HOME())); + CoreOptions.APP_HOME.key())); } - Properties properties = new Properties(); File file = new File(appHome, "conf/config.yaml"); if (file.exists() && file.isFile()) { - Map config = PropertiesUtils.fromYamlFileAsJava(file.getAbsolutePath()); - properties.putAll(config); - return properties; + return ConfigurationParser.parse(Path.of(file.getAbsolutePath())); } else { throw new ExceptionInInitializerError(file.getAbsolutePath() + " not found, please check."); } @@ -173,4 +187,7 @@ private static Properties getSpringConfig() { config.put("management.health.ldap.enabled", "false"); return config; } + + private SpringProperties() { + } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/enums/CatalogType.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/package-info.java similarity index 82% rename from streampark-common/src/main/java/org/apache/streampark/common/enums/CatalogType.java rename to streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/package-info.java index d3baa95d4f..bec935706c 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/enums/CatalogType.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/package-info.java @@ -15,15 +15,5 @@ * limitations under the License. */ -package org.apache.streampark.common.enums; - -/** catalog type */ -public enum CatalogType { - MYSQL, - PGSQL, - ORACLE, - JDBC, - HIVE, - PAIMON, - CUSTOM -} +/** Typed configuration options owned by the StreamPark Console process. */ +package org.apache.streampark.console.base.config; diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java index adc87f2dd8..5b7dfab152 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java @@ -17,9 +17,10 @@ package org.apache.streampark.console.base.util; -import org.apache.streampark.common.conf.FlinkVersion; +import org.apache.streampark.common.configuration.ConfigurationParser; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.util.FileUtils; -import org.apache.streampark.common.util.PropertiesUtils; +import org.apache.streampark.common.util.FlinkConfigurationUtils; import org.apache.commons.io.output.NullOutputStream; @@ -49,7 +50,7 @@ public static void main(String[] args) throws IOException { case "--get_yaml": String key = actionArgs[0]; String conf = actionArgs[1]; - Map confMap = PropertiesUtils.fromYamlFileAsJava(conf); + Map confMap = ConfigurationParser.parse(Paths.get(conf)).toMap(); String value = confMap.get(key); System.out.println(value); break; @@ -82,10 +83,9 @@ public static void main(String[] args) throws IOException { System.setOut(new PrintStream(new NullOutputStream())); String version = flinkVersion.majorVersion(); - float ver = Float.parseFloat(version); - File yaml = new File(flinkHome, ver < 1.19f ? "/conf/flink-conf.yaml" : "/conf/config.yaml"); - - Map config = PropertiesUtils.fromYamlFileAsJava(yaml.getAbsolutePath()); + Map config = + FlinkConfigurationUtils.loadConfiguration( + new File(flinkHome, "conf").getAbsolutePath(), version); String flinkPort = config.getOrDefault("rest.port", "8081"); System.setOut(originalOut); System.out.println( diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java index 84da042c2d..cda738879e 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java @@ -17,26 +17,69 @@ package org.apache.streampark.console.base.util; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.option.CoreOptions; +import org.apache.streampark.common.util.OkHttpUtils; import org.apache.commons.lang3.StringUtils; import com.baomidou.mybatisplus.core.toolkit.StringPool; -import lombok.extern.slf4j.Slf4j; +import okhttp3.Headers; +import okhttp3.MediaType; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.springframework.http.HttpMethod; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.Enumeration; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; import java.util.stream.IntStream; -/** Web Utils */ -@Slf4j +/** Common web-path helpers and servlet proxy support for the Console. */ public final class WebUtils { + /** Inbound connection headers and metadata reconstructed by OkHttp. */ + private static final Set REQUEST_SKIP_HEADERS = + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "content-length"))); + + private static final Set RESPONSE_SKIP_HEADERS = + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "content-length"))); + private static final String TEMP = "temp"; private static final String LIB = "lib"; private static final String PLUGINS = "plugins"; private static final String CLIENT = "client"; - private static final String CONF = "conf"; - private WebUtils() { } @@ -68,7 +111,7 @@ public static String camelToUnderscore(String value) { } public static String getAppHome() { - return System.getProperty(ConfigKeys.KEY_APP_HOME()); + return System.getProperty(CoreOptions.APP_HOME.key()); } public static File getAppDir(String dir) { @@ -91,4 +134,99 @@ public static File getPluginDir() { return getAppDir(PLUGINS); } + /** Proxies a request and streams the upstream response to the servlet response. */ + public static void http( + String url, + HttpServletRequest request, + HttpServletResponse response) throws IOException { + Headers.Builder headersBuilder = new Headers.Builder(); + Set skippedHeaders = + skippedHeaders(request.getHeader("Connection"), REQUEST_SKIP_HEADERS); + Enumeration headerNames = request.getHeaderNames(); + if (headerNames != null) { + while (headerNames.hasMoreElements()) { + String headerName = headerNames.nextElement(); + String normalizedName = headerName.toLowerCase(Locale.ROOT); + if ("referer".equals(normalizedName) + || "origin".equals(normalizedName) + || "host".equals(normalizedName) + || skippedHeaders.contains(normalizedName) + || normalizedName.startsWith("access-control-")) { + continue; + } + Enumeration values = request.getHeaders(headerName); + if (values == null) { + continue; + } + while (values.hasMoreElements()) { + String value = values.nextElement(); + if (value != null) { + headersBuilder.add(headerName, value); + } + } + } + } + + Request.Builder requestBuilder = + new Request.Builder().url(url).headers(headersBuilder.build()); + // OkHttp derives Host and Content-Length from the upstream URL and replayable request body. + String method = request.getMethod(); + if (HttpMethod.GET.matches(method) || HttpMethod.HEAD.matches(method)) { + requestBuilder.method(method, null); + } else { + byte[] content = org.apache.commons.io.IOUtils.toByteArray(request.getInputStream()); + MediaType contentType = + StringUtils.isNotBlank(request.getContentType()) + ? MediaType.parse(request.getContentType()) + : null; + requestBuilder.method(method, RequestBody.create(content, contentType)); + } + + try (Response upstream = OkHttpUtils.call(requestBuilder.build())) { + writeProxyResponse(upstream, response); + } + } + + private static void writeProxyResponse( + Response upstream, HttpServletResponse downstream) throws IOException { + copyProxyResponseHeaders(upstream, downstream); + downstream.setStatus(upstream.code()); + downstream.setHeader("Access-Control-Allow-Origin", "*"); + if (HttpMethod.HEAD.matches(upstream.request().method())) { + return; + } + ResponseBody body = upstream.body(); + if (body != null) { + org.apache.commons.io.IOUtils.copy(body.byteStream(), downstream.getOutputStream()); + } + } + + private static void copyProxyResponseHeaders( + Response upstream, + HttpServletResponse downstream) { + Set skippedHeaders = + skippedHeaders(upstream.header("Connection"), RESPONSE_SKIP_HEADERS); + upstream.headers().forEach( + header -> { + String normalizedName = header.getFirst().toLowerCase(Locale.ROOT); + if (!normalizedName.startsWith("access-control-") + && !skippedHeaders.contains(normalizedName)) { + downstream.addHeader(header.getFirst(), header.getSecond()); + } + }); + } + + /** Adds header names declared by {@code Connection} to the fixed exclusion set. */ + private static Set skippedHeaders(String connection, Set fixedHeaders) { + Set result = new HashSet<>(fixedHeaders); + if (connection != null) { + Arrays.stream(connection.split(",")) + .map(String::trim) + .filter(value -> !value.isEmpty()) + .map(value -> value.toLowerCase(Locale.ROOT)) + .forEach(result::add); + } + return result; + } + } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/assembler/FlinkEnvAssembler.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/assembler/FlinkEnvAssembler.java index 79b16f36c5..db61226850 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/assembler/FlinkEnvAssembler.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/assembler/FlinkEnvAssembler.java @@ -24,6 +24,7 @@ import org.apache.streampark.console.core.request.flink.FlinkEnvPageQueryRequest; import org.apache.streampark.console.core.request.flink.FlinkEnvUpdateRequest; import org.apache.streampark.console.core.response.flink.FlinkEnvResponse; +import org.apache.streampark.console.core.util.FlinkEnvUtils; import com.baomidou.mybatisplus.core.metadata.IPage; import org.springframework.beans.BeanUtils; @@ -96,6 +97,15 @@ public static FlinkEnvResponse toResponse(FlinkEnv env) { return response; } + /** Converts an entity to a detail response with readable YAML configuration. */ + public static FlinkEnvResponse toDetailResponse(FlinkEnv env) { + FlinkEnvResponse response = toResponse(env); + if (response != null) { + response.setFlinkConf(FlinkEnvUtils.yaml(env)); + } + return response; + } + public static List toListResponse(List envs) { if (envs == null) { return Collections.emptyList(); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/Dependency.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/Dependency.java index 6a743c4e92..7c49c85137 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/Dependency.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/Dependency.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.core.bean; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.util.FileUtils; import org.apache.streampark.common.util.Utils; import org.apache.streampark.console.base.util.JacksonUtils; @@ -67,7 +67,7 @@ public boolean equals(Dependency other) { return false; } File localJar = WebUtils.getAppTempDir(); - File localUploads = new File(Workspace.local().APP_UPLOADS()); + File localUploads = new File(Workspace.LOCAL.uploads); Set otherJars = new HashSet<>(other.jar); for (String jarName : jar) { if (!otherJars.contains(jarName) @@ -81,7 +81,7 @@ public boolean equals(Dependency other) { public DependencyInfo toJarPackDeps() { List mvnArts = toArtifact(); List extJars = this.jar.stream() - .map(jar -> Workspace.local().APP_UPLOADS() + "/" + jar) + .map(jar -> Workspace.LOCAL.uploads + "/" + jar) .collect(Collectors.toList()); return new DependencyInfo(mvnArts, extJars); } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/MavenConfig.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/MavenConfig.java index 058dc2785c..d0fc91b412 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/MavenConfig.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/MavenConfig.java @@ -17,8 +17,8 @@ package org.apache.streampark.console.core.bean; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.InternalConfigHolder; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.option.MavenOptions; import org.apache.streampark.console.core.entity.Setting; import org.apache.streampark.console.core.service.SettingService; @@ -49,28 +49,28 @@ public class MavenConfig { /** Password for Maven authentication. */ private String mvnAuthPassword; - /** */ + /** Creates Maven configuration from settings stored by the console service. */ public static MavenConfig fromSetting() { MavenConfig mavenConfig = new MavenConfig(); Map settings = SettingService.SETTINGS; - if (settings.containsKey(CommonConfig.MAVEN_SETTINGS_PATH().key())) { + if (settings.containsKey(MavenOptions.SETTINGS_PATH.key())) { mavenConfig.setMvnSettings( - settings.get(CommonConfig.MAVEN_SETTINGS_PATH().key()).getSettingValue()); + settings.get(MavenOptions.SETTINGS_PATH.key()).getSettingValue()); } - if (settings.containsKey(CommonConfig.MAVEN_REMOTE_URL().key())) { + if (settings.containsKey(MavenOptions.REPOSITORY_URL.key())) { mavenConfig.setMvnRepository( - settings.get(CommonConfig.MAVEN_REMOTE_URL().key()).getSettingValue()); + settings.get(MavenOptions.REPOSITORY_URL.key()).getSettingValue()); } - if (settings.containsKey(CommonConfig.MAVEN_AUTH_USER().key())) { + if (settings.containsKey(MavenOptions.USER_NAME.key())) { mavenConfig.setMvnAuthUser( - settings.get(CommonConfig.MAVEN_AUTH_USER().key()).getSettingValue()); + settings.get(MavenOptions.USER_NAME.key()).getSettingValue()); } - if (settings.containsKey(CommonConfig.MAVEN_AUTH_PASSWORD().key())) { + if (settings.containsKey(MavenOptions.PASSWORD.key())) { mavenConfig.setMvnAuthPassword( - settings.get(CommonConfig.MAVEN_AUTH_PASSWORD().key()).getSettingValue()); + settings.get(MavenOptions.PASSWORD.key()).getSettingValue()); } return mavenConfig; @@ -84,19 +84,19 @@ public static MavenConfig fromSetting() { public void updateConfig() { if (StringUtils.isNotBlank(mvnSettings)) { - InternalConfigHolder.set(CommonConfig.MAVEN_SETTINGS_PATH(), mvnSettings); + GlobalConfiguration.set(MavenOptions.SETTINGS_PATH, mvnSettings, "database setting"); } if (StringUtils.isNotBlank(mvnRepository)) { - InternalConfigHolder.set(CommonConfig.MAVEN_REMOTE_URL(), mvnRepository); + GlobalConfiguration.set(MavenOptions.REPOSITORY_URL, mvnRepository, "database setting"); } if (StringUtils.isNotBlank(mvnAuthUser)) { - InternalConfigHolder.set(CommonConfig.MAVEN_AUTH_USER(), mvnAuthUser); + GlobalConfiguration.set(MavenOptions.USER_NAME, mvnAuthUser, "database setting"); } if (StringUtils.isNotBlank(mvnAuthPassword)) { - InternalConfigHolder.set(CommonConfig.MAVEN_AUTH_PASSWORD(), mvnAuthPassword); + GlobalConfiguration.set(MavenOptions.PASSWORD, mvnAuthPassword, "database setting"); } } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/FlinkEnvController.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/FlinkEnvController.java index 69963e2786..902c5d87ef 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/FlinkEnvController.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/FlinkEnvController.java @@ -82,8 +82,7 @@ public RestResponseBody create(@Valid @FormOrJson FlinkEnvCreateRequest public RestResponseBody get(@Valid IdRequest request) throws Exception { FlinkEnv flinkEnv = flinkEnvService.getById(request.getId()); ApiAlertException.throwIfNull(flinkEnv, "Flink environment not found."); - flinkEnv.unzipFlinkConf(); - return RestResponseBody.success(FlinkEnvAssembler.toResponse(flinkEnv)); + return RestResponseBody.success(FlinkEnvAssembler.toDetailResponse(flinkEnv)); } @PostMapping("sync") diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProxyController.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProxyController.java index a449cc6670..e117db4fb3 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProxyController.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProxyController.java @@ -36,7 +36,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.ResponseEntity; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -44,6 +43,7 @@ import org.springframework.web.bind.annotation.RestController; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; @Slf4j @Validated @@ -67,41 +67,47 @@ public class ProxyController { private MemberService memberService; @GetMapping("{type}/{id}/assets/**") - public ResponseEntity proxyFlinkAssets(HttpServletRequest request, @PathVariable("type") String type, - @PathVariable("id") Long id) throws Exception { - return proxy(type, request, id); + public void proxyFlinkAssets(HttpServletRequest request, HttpServletResponse response, + @PathVariable("type") String type, @PathVariable("id") Long id) throws Exception { + proxy(type, request, response, id); } @GetMapping("{type}/{id}/**") @RequiresPermissions("app:view") - public ResponseEntity proxyFlink(HttpServletRequest request, @PathVariable("type") String type, - @PathVariable("id") Long id) throws Exception { - return proxy(type, request, id); + public void proxyFlink(HttpServletRequest request, HttpServletResponse response, + @PathVariable("type") String type, @PathVariable("id") Long id) throws Exception { + proxy(type, request, response, id); } - private ResponseEntity proxy(String type, HttpServletRequest request, Long id) throws Exception { + private void proxy(String type, HttpServletRequest request, HttpServletResponse response, + Long id) throws Exception { ApplicationLog log; switch (type) { case "flink": FlinkApplication flinkApplication = flinkApplicationManageService.getApp(id); checkProxyApp(flinkApplication.getTeamId()); - return proxyService.proxyFlink(request, flinkApplication); + proxyService.proxyFlink(request, response, flinkApplication); + return; case "spark": SparkApplication sparkApplication = sparkApplicationManageService.getApp(id); checkProxyApp(sparkApplication.getTeamId()); - return proxyService.proxySpark(request, sparkApplication); + proxyService.proxySpark(request, response, sparkApplication); + return; case "flink_cluster": - return proxyService.proxyFlinkCluster(request, id); + proxyService.proxyFlinkCluster(request, response, id); + return; case "history": log = logService.getById(id); checkProxyAppLog(log); - return proxyService.proxyHistory(request, log); + proxyService.proxyHistory(request, response, log); + return; case "yarn": log = logService.getById(id); checkProxyAppLog(log); - return proxyService.proxyYarn(request, log); + proxyService.proxyYarn(request, response, log); + return; default: - return ResponseEntity.notFound().build(); + response.sendError(HttpServletResponse.SC_NOT_FOUND); } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplication.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplication.java index f16dfc675f..6ffedd72b3 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplication.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplication.java @@ -17,12 +17,12 @@ package org.apache.streampark.console.core.entity; -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.option.DeploymentOptions; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; -import org.apache.streampark.common.enums.FlinkK8sRestExposedType; +import org.apache.streampark.common.enums.FlinkKubernetesRestExposedType; import org.apache.streampark.common.enums.StorageType; import org.apache.streampark.console.base.mybatis.entity.BaseEntity; import org.apache.streampark.console.base.util.JacksonUtils; @@ -32,8 +32,9 @@ import org.apache.streampark.console.core.enums.ReleaseStateEnum; import org.apache.streampark.console.core.enums.ResourceFromEnum; import org.apache.streampark.console.core.metrics.flink.JobsOverview; +import org.apache.streampark.console.core.util.ApplicationEntitySupport; import org.apache.streampark.console.core.util.YarnQueueLabelExpression; -import org.apache.streampark.flink.kubernetes.model.K8sPodTemplates; +import org.apache.streampark.flink.kubernetes.model.KubernetesPodTemplates; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -63,10 +64,6 @@ @Slf4j public class FlinkApplication extends BaseEntity implements ApplicationEntitySupport, ReleaseOutcomeTarget { - private static final TypeReference> STRING_OBJECT_MAP = - new TypeReference>() { - }; - @TableId(type = IdType.AUTO) private Long id; @@ -322,8 +319,8 @@ public void setK8sNamespace(String k8sNamespace) { this.k8sNamespace = StringUtils.isBlank(k8sNamespace) ? Constants.DEFAULT : k8sNamespace; } - public K8sPodTemplates getK8sPodTemplates() { - return K8sPodTemplates.of(k8sPodTemplate, k8sJmPodTemplate, k8sTmPodTemplate); + public KubernetesPodTemplates getK8sPodTemplates() { + return KubernetesPodTemplates.of(k8sPodTemplate, k8sJmPodTemplate, k8sTmPodTemplate); } public void setState(Integer state) { @@ -339,9 +336,9 @@ public void setYarnQueueByHotParams() { Map hotParamsMap = this.getHotParamsMap(); if (MapUtils.isNotEmpty(hotParamsMap) - && hotParamsMap.containsKey(ConfigKeys.KEY_YARN_APP_QUEUE())) { - String yarnQueue = hotParamsMap.get(ConfigKeys.KEY_YARN_APP_QUEUE()).toString(); - String labelExpr = Optional.ofNullable(hotParamsMap.get(ConfigKeys.KEY_YARN_APP_NODE_LABEL())) + && hotParamsMap.containsKey(DeploymentOptions.YARN_QUEUE.key())) { + String yarnQueue = hotParamsMap.get(DeploymentOptions.YARN_QUEUE.key()).toString(); + String labelExpr = Optional.ofNullable(hotParamsMap.get(DeploymentOptions.YARN_NODE_LABEL.key())) .map(Object::toString) .orElse(null); this.setYarnQueue(YarnQueueLabelExpression.of(yarnQueue, labelExpr).toString()); @@ -407,8 +404,8 @@ public FlinkAppStateEnum getStateEnum() { } @JsonIgnore - public FlinkK8sRestExposedType getK8sRestExposedTypeEnum() { - return FlinkK8sRestExposedType.of(this.k8sRestExposedType); + public FlinkKubernetesRestExposedType getK8sRestExposedTypeEnum() { + return FlinkKubernetesRestExposedType.of(this.k8sRestExposedType); } @JsonIgnore @@ -474,13 +471,15 @@ public Map getOptionMap() { if (StringUtils.isBlank(this.options)) { return new HashMap<>(); } - Map optionMap = JacksonUtils.read(this.options, STRING_OBJECT_MAP); + Map optionMap = JacksonUtils.read(this.options, new TypeReference<>() { + }); optionMap.entrySet().removeIf(entry -> entry.getValue() == null); return optionMap; } + /** Returns whether this application runs Flink SQL. */ @JsonIgnore - public boolean isFlinkSql() { + public boolean isFlinkSqlJob() { return FlinkJobType.FLINK_SQL.getMode().equals(this.getJobType()); } @@ -547,7 +546,8 @@ public static StorageType getStorageType(Integer deployMode) { @SneakyThrows public Map getHotParamsMap() { if (this.hotParams != null) { - Map map = JacksonUtils.read(this.hotParams, STRING_OBJECT_MAP); + Map map = JacksonUtils.read(this.hotParams, new TypeReference<>() { + }); map.entrySet().removeIf(entry -> entry.getValue() == null); return map; } @@ -571,7 +571,9 @@ public void updateHotParams(FlinkApplication appParam) { } if (deployModeEnum == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) { if (StringUtils.isNotBlank(appParam.getServiceAccount())) { - hotParams.put(ConfigKeys.KEY_KERBEROS_SERVICE_ACCOUNT(), appParam.getServiceAccount()); + hotParams.put( + DeploymentOptions.KUBERNETES_SERVICE_ACCOUNT.key(), + appParam.getServiceAccount()); } } if (!hotParams.isEmpty()) { diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplicationBackup.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplicationBackup.java index c30785abb0..a4fa1265bb 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplicationBackup.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplicationBackup.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.core.entity; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.Workspace; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; @@ -73,12 +73,12 @@ private void renderPath(FlinkApplication application) { case LOCAL: this.path = String.format( "%s/%d/%d", - Workspace.local().APP_BACKUPS(), application.getId(), createTime.getTime()); + Workspace.LOCAL.backups, application.getId(), createTime.getTime()); break; case YARN_APPLICATION: this.path = String.format( "%s/%d/%d", - Workspace.remote().APP_BACKUPS(), application.getId(), createTime.getTime()); + Workspace.REMOTE.backups, application.getId(), createTime.getTime()); break; default: throw new UnsupportedOperationException( diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplicationConfig.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplicationConfig.java index 68b953f530..686c0cfc50 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplicationConfig.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkApplicationConfig.java @@ -17,13 +17,6 @@ package org.apache.streampark.console.core.entity; -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.PropertiesUtils; -import org.apache.streampark.console.core.enums.ConfigFileTypeEnum; - -import org.apache.commons.collections.MapUtils; - import com.baomidou.mybatisplus.annotation.FieldStrategy; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; @@ -31,20 +24,13 @@ import com.baomidou.mybatisplus.annotation.TableName; import lombok.Getter; import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import javax.annotation.Nullable; - -import java.util.Base64; import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.stream.Collectors; +/** Persistent version of an application's user-supplied Flink configuration. */ @Getter @Setter @TableName("t_flink_config") -@Slf4j public class FlinkApplicationConfig { @TableId(type = IdType.AUTO) @@ -71,77 +57,4 @@ public class FlinkApplicationConfig { private Boolean latest; private transient boolean effective = false; - - public void setToApplication(FlinkApplication application) { - String unzipString = DeflaterUtils.unzipString(content); - String encode = Base64.getEncoder().encodeToString(unzipString.getBytes()); - application.setConfig(encode); - application.setConfigId(this.id); - application.setFormat(this.format); - } - - public void setToApplication(SparkApplication application) { - String unzipString = DeflaterUtils.unzipString(content); - String encode = Base64.getEncoder().encodeToString(unzipString.getBytes()); - application.setConfig(encode); - application.setConfigId(this.id); - application.setFormat(this.format); - } - - public Map readConfig() { - Map configs = renderConfigs(); - - if (MapUtils.isNotEmpty(configs)) { - return configs.entrySet().stream() - .collect( - Collectors.toMap( - entry -> { - String key = entry.getKey(); - if (key.startsWith( - ConfigKeys.KEY_FLINK_OPTION_PREFIX())) { - key = key.substring( - ConfigKeys.KEY_FLINK_OPTION_PREFIX() - .length()); - } else if (key.startsWith(ConfigKeys - .KEY_FLINK_PROPERTY_PREFIX())) { - key = key.substring(ConfigKeys - .KEY_FLINK_PROPERTY_PREFIX() - .length()); - } else if (key.startsWith( - ConfigKeys.KEY_FLINK_TABLE_PREFIX())) { - key = key.substring(ConfigKeys - .KEY_FLINK_TABLE_PREFIX().length()); - } else if (key.startsWith( - ConfigKeys.KEY_APP_PREFIX())) { - key = key.substring(ConfigKeys.KEY_APP_PREFIX() - .length()); - } else if (key.startsWith( - ConfigKeys.KEY_SQL_PREFIX())) { - key = key.substring(ConfigKeys.KEY_SQL_PREFIX() - .length()); - } - return key; - }, - Map.Entry::getValue)); - } - return new HashMap<>(); - } - - @Nullable - private Map renderConfigs() { - ConfigFileTypeEnum fileType = ConfigFileTypeEnum.of(this.format); - if (fileType == null) { - return null; - } - switch (fileType) { - case YAML: - return PropertiesUtils.fromYamlTextAsJava(DeflaterUtils.unzipString(this.content)); - case PROPERTIES: - return PropertiesUtils.fromPropertiesTextAsJava(DeflaterUtils.unzipString(this.content)); - case HOCON: - return PropertiesUtils.fromHoconTextAsJava(DeflaterUtils.unzipString(this.content)); - default: - return new HashMap<>(); - } - } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkCluster.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkCluster.java index b50747ce0a..20caf47d19 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkCluster.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkCluster.java @@ -17,12 +17,12 @@ package org.apache.streampark.console.core.entity; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.JvmOptionsParser; +import org.apache.streampark.common.configuration.option.DeploymentOptions; import org.apache.streampark.common.enums.ClusterState; import org.apache.streampark.common.enums.FlinkDeployMode; -import org.apache.streampark.common.enums.FlinkK8sRestExposedType; +import org.apache.streampark.common.enums.FlinkKubernetesRestExposedType; import org.apache.streampark.common.enums.ResolveOrder; -import org.apache.streampark.common.util.FlinkConfigurationUtils; import org.apache.streampark.common.util.HttpClientUtils; import org.apache.streampark.console.base.util.JacksonUtils; import org.apache.streampark.console.core.util.YarnQueueLabelExpression; @@ -120,8 +120,8 @@ public class FlinkCluster implements Serializable { private transient Integer affectedJobs = 0; @JsonIgnore - public FlinkK8sRestExposedType getK8sRestExposedTypeEnum() { - return FlinkK8sRestExposedType.of(this.k8sRestExposedType); + public FlinkKubernetesRestExposedType getK8sRestExposedTypeEnum() { + return FlinkKubernetesRestExposedType.of(this.k8sRestExposedType); } @JsonIgnore @@ -142,7 +142,7 @@ public Map getOptionMap() { } Map optionMap = JacksonUtils.read(this.options, Map.class); if (FlinkDeployMode.YARN_SESSION == getFlinkDeployModeEnum()) { - optionMap.put(ConfigKeys.KEY_YARN_APP_NAME(), this.clusterName); + optionMap.put(DeploymentOptions.YARN_APPLICATION_NAME.key(), this.clusterName); optionMap.putAll(YarnQueueLabelExpression.getQueueLabelMap(yarnQueue)); } optionMap.entrySet().removeIf(entry -> entry.getValue() == null); @@ -181,7 +181,7 @@ public Map getFlinkConfig() throws JsonProcessingException { public Map getProperties() { Map propertyMap = new HashMap<>(); Map dynamicPropertyMap = - FlinkConfigurationUtils.extractDynamicPropertiesAsJava(this.getDynamicProperties()); + JvmOptionsParser.parse(this.getDynamicProperties()).toMap(); propertyMap.putAll(this.getOptionMap()); propertyMap.putAll(dynamicPropertyMap); ResolveOrder resolveOrder = ResolveOrder.of(this.getResolveOrder()); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkEnv.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkEnv.java index 66942473b5..8ad1c1e55c 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkEnv.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkEnv.java @@ -17,29 +17,16 @@ package org.apache.streampark.console.core.entity; -import org.apache.streampark.common.conf.FlinkVersion; -import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.FlinkConfigurationUtils; -import org.apache.streampark.console.base.exception.ApiAlertException; -import org.apache.streampark.console.base.exception.ApiDetailException; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.StringUtils; - import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; -import java.io.File; import java.io.Serializable; -import java.nio.charset.StandardCharsets; import java.util.Date; -import java.util.Map; -import java.util.Properties; +/** Persistent metadata for a locally installed Flink distribution. */ @Getter @Setter @TableName("t_flink_env") @@ -60,127 +47,8 @@ public class FlinkEnv implements Serializable { private String version; - /** is default */ + /** Whether this installation is the default Flink environment. */ private Boolean isDefault; private Date createTime; - - private transient FlinkVersion flinkVersion; - - private transient String versionOfLarge; - - private transient String versionOfMiddle; - - private transient String versionOfLast; - - private transient String streamParkScalaVersion = scala.util.Properties.versionNumberString(); - - public void doSetFlinkConf() throws ApiDetailException { - File yaml; - Float ver = getVersionNumber(); - if (ver < 1.19f) { - yaml = new File(this.flinkHome.concat("/conf/flink-conf.yaml")); - if (!yaml.exists()) { - throw new ApiAlertException("cannot find flink-conf.yaml in flink/conf "); - } - } else if (ver == 1.19f) { - yaml = new File(this.flinkHome.concat("/conf/flink-conf.yaml")); - if (!yaml.exists()) { - yaml = new File(this.flinkHome.concat("/conf/config.yaml")); - } - if (!yaml.exists()) { - throw new ApiAlertException("cannot find config.yaml|flink-conf.yaml in flink/conf "); - } - } else { - yaml = new File(this.flinkHome.concat("/conf/config.yaml")); - if (!yaml.exists()) { - throw new ApiAlertException("cannot find config.yaml in flink/conf "); - } - } - try { - String flinkConf = FileUtils.readFileToString(yaml, StandardCharsets.UTF_8); - this.flinkConf = DeflaterUtils.zipString(flinkConf); - } catch (Exception e) { - throw new ApiDetailException(e); - } - } - - public void doSetVersion() { - this.setVersion(this.getFlinkVersion().version()); - this.setScalaVersion(this.getFlinkVersion().scalaVersion()); - if (!streamParkScalaVersion.startsWith(this.getFlinkVersion().scalaVersion())) { - throw new UnsupportedOperationException( - String.format( - "The current Scala version of StreamPark is %s, but the scala version of Flink to be added is %s, which does not match, Please check", - streamParkScalaVersion, this.getFlinkVersion().scalaVersion())); - } - } - - public Map convertFlinkYamlAsMap() { - String flinkYamlString = DeflaterUtils.unzipString(flinkConf); - if (isLegacyFlinkConf()) { - return FlinkConfigurationUtils.loadLegacyFlinkConf(flinkYamlString); - } - return FlinkConfigurationUtils.loadFlinkConf(flinkYamlString); - } - - @JsonIgnore - public FlinkVersion getFlinkVersion() { - if (this.flinkVersion == null) { - this.flinkVersion = new FlinkVersion(this.flinkHome); - } - return this.flinkVersion; - } - - public void unzipFlinkConf() { - this.flinkConf = DeflaterUtils.unzipString(this.flinkConf); - } - - public String getLargeVersion() { - if (StringUtils.isNotBlank(this.version)) { - return this.version.substring(0, this.version.lastIndexOf(".")); - } - return null; - } - - public String getVersionOfFirst() { - if (StringUtils.isNotBlank(this.version)) { - return this.version.split("\\.")[0]; - } - return null; - } - - public String getVersionOfMiddle() { - if (StringUtils.isNotBlank(this.version)) { - return this.version.split("\\.")[1]; - } - return null; - } - - public String getVersionOfLast() { - if (StringUtils.isNotBlank(this.version)) { - return this.version.split("\\.")[2]; - } - return null; - } - - @JsonIgnore - public Properties getFlinkConfig() { - String flinkYamlString = DeflaterUtils.unzipString(flinkConf); - Properties flinkConfig = new Properties(); - Map config = FlinkConfigurationUtils.loadLegacyFlinkConf(flinkYamlString); - flinkConfig.putAll(config); - return flinkConfig; - } - - private Float getVersionNumber() { - if (StringUtils.isNotBlank(this.version)) { - return Float.parseFloat(getVersionOfFirst() + "." + getVersionOfMiddle()); - } - throw new IllegalStateException("Flink version is null"); - } - - public boolean isLegacyFlinkConf() { - return getVersionNumber() < 1.19f; - } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkSql.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkSql.java index 508a50344f..e986ac33b1 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkSql.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkSql.java @@ -30,6 +30,7 @@ import lombok.Getter; import lombok.Setter; +import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Date; import java.util.Objects; @@ -86,37 +87,42 @@ public FlinkSql(SparkApplication application) { this.createTime = new Date(); } + /** + * Converts the persisted SQL representation to plain text. + * + * @throws IllegalStateException if the persisted value is not valid compressed SQL + */ public void decode() { - this.setSql(DeflaterUtils.toPlainText(this.sql)); - } - - public void setToApplication(FlinkApplication application) { - String encode = Base64.getEncoder().encodeToString(this.sql.getBytes()); - application.setFlinkSql(encode); - application.setDependency(this.dependency); - application.setTeamResource(this.teamResource); - application.setSqlId(this.id); + if (this.sql == null) { + return; + } + String decodedSql = DeflaterUtils.unzipString(this.sql); + if (decodedSql == null) { + throw new IllegalStateException("Failed to decompress Flink SQL id=" + this.id); + } + this.sql = decodedSql; } - public void setToApplication(SparkApplication application) { - String encode = Base64.getEncoder().encodeToString(this.sql.getBytes()); - application.setSparkSql(encode); + /** Copies decoded SQL to a Flink application using the API transport representation. */ + public void applyToApplication(FlinkApplication application) { + application.setFlinkSql( + Base64.getEncoder().encodeToString(this.sql.getBytes(StandardCharsets.UTF_8))); application.setDependency(this.dependency); application.setTeamResource(this.teamResource); application.setSqlId(this.id); } + /** Compares decoded SQL and dependency metadata with another version. */ public ChangeTypeEnum checkChange(FlinkSql target) { if (target == null) { return ChangeTypeEnum.NONE; } // 1) determine if sql statement has changed - String sourceSql = - StringUtils.trimToEmpty(DeflaterUtils.toPlainText(this.getSql())); + String sourceSql = StringUtils.trimToEmpty(this.getSql()); String targetSql = target.getSql() == null ? sourceSql - : StringUtils.trimToEmpty(DeflaterUtils.toPlainText(target.getSql())); + : StringUtils.trimToEmpty(target.getSql()); boolean sqlDifference = !sourceSql.equals(targetSql); // 2) determine if dependency has changed Dependency thisDependency = Dependency.toDependency(this.getDependency()); @@ -140,7 +146,10 @@ public ChangeTypeEnum checkChange(FlinkSql target) { return ChangeTypeEnum.NONE; } + /** Replaces SQL with the Base64-encoded plain text required by the API. */ public void base64Encode() { - this.sql = Base64.getEncoder().encodeToString(DeflaterUtils.unzipString(this.sql).getBytes()); + decode(); + this.sql = + Base64.getEncoder().encodeToString(this.sql.getBytes(StandardCharsets.UTF_8)); } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java index 79a23630eb..5da8e3dc2e 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java @@ -17,9 +17,9 @@ package org.apache.streampark.console.core.entity; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.InternalConfigHolder; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.Workspace; +import org.apache.streampark.common.configuration.option.MavenOptions; import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.Utils; import org.apache.streampark.console.base.util.WebUtils; @@ -106,7 +106,7 @@ public class Project implements Serializable { /** get project source */ @JsonIgnore public File getAppSource() { - File sourcePath = new File(Workspace.PROJECT_LOCAL_PATH()); + File sourcePath = new File(Workspace.PROJECT_LOCAL_PATH); if (!sourcePath.exists()) { sourcePath.mkdirs(); } else if (sourcePath.isFile()) { @@ -148,7 +148,7 @@ private String getSourceDirName() { @JsonIgnore public File getDistHome() { - return new File(Workspace.APP_LOCAL_DIST(), id.toString()); + return new File(Workspace.APP_LOCAL_DIST, id.toString()); } @JsonIgnore @@ -187,7 +187,8 @@ public String getMavenArgs() { } // --settings - String setting = InternalConfigHolder.get(CommonConfig.MAVEN_SETTINGS_PATH()); + String setting = + GlobalConfiguration.current().getOptional(MavenOptions.SETTINGS_PATH).orElse(null); if (StringUtils.isNotBlank(setting)) { File file = new File(setting); if (file.exists() && file.isFile()) { diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplication.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplication.java index 1719279386..87dc2c1239 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplication.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplication.java @@ -17,8 +17,8 @@ package org.apache.streampark.console.core.entity; -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.option.DeploymentOptions; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.SparkDeployMode; import org.apache.streampark.common.enums.SparkJobType; @@ -31,8 +31,10 @@ import org.apache.streampark.console.core.enums.ResourceFromEnum; import org.apache.streampark.console.core.enums.SparkAppStateEnum; import org.apache.streampark.console.core.metrics.spark.SparkApplicationSummary; +import org.apache.streampark.console.core.util.ApplicationEntitySupport; import org.apache.streampark.console.core.util.YarnQueueLabelExpression; -import org.apache.streampark.spark.kubernetes.model.SparkK8sPodTemplates; +import org.apache.streampark.spark.configuration.SparkOptions; +import org.apache.streampark.spark.kubernetes.model.SparkKubernetesPodTemplates; import org.apache.commons.lang3.StringUtils; @@ -248,8 +250,10 @@ public void resolveYarnQueue() { this.yarnQueue = "default"; } Map queueLabelMap = YarnQueueLabelExpression.getQueueLabelMap(this.yarnQueue); - this.setYarnQueueName(queueLabelMap.getOrDefault(ConfigKeys.KEY_YARN_APP_QUEUE(), "default")); - this.setYarnQueueLabel(queueLabelMap.getOrDefault(ConfigKeys.KEY_YARN_APP_NODE_LABEL(), null)); + this.setYarnQueueName( + queueLabelMap.getOrDefault(DeploymentOptions.YARN_QUEUE.key(), "default")); + this.setYarnQueueLabel( + queueLabelMap.getOrDefault(DeploymentOptions.YARN_NODE_LABEL.key(), null)); } /** @@ -264,16 +268,17 @@ public void resolveYarnQueue() { * configuration template and custom configurations. */ public void resolveScheduleConf(Map map) { - this.setDriverCores(map.get(ConfigKeys.KEY_SPARK_DRIVER_CORES())); - this.setDriverMemory(map.get(ConfigKeys.KEY_SPARK_DRIVER_MEMORY())); - this.setExecutorCores(map.get(ConfigKeys.KEY_SPARK_EXECUTOR_CORES())); - this.setExecutorMemory(map.get(ConfigKeys.KEY_SPARK_EXECUTOR_MEMORY())); + this.setDriverCores(map.get(SparkOptions.DRIVER_CORES.key())); + this.setDriverMemory(map.get(SparkOptions.DRIVER_MEMORY.key())); + this.setExecutorCores(map.get(SparkOptions.EXECUTOR_CORES.key())); + this.setExecutorMemory(map.get(SparkOptions.EXECUTOR_MEMORY.key())); boolean isDynamicAllocationEnabled = - Boolean.parseBoolean(map.get(ConfigKeys.KEY_SPARK_DYNAMIC_ALLOCATION_ENABLED())); + Boolean.parseBoolean(map.get(SparkOptions.DYNAMIC_ALLOCATION_ENABLED.key())); if (isDynamicAllocationEnabled) { - this.setExecutorMaxNums(map.getOrDefault(ConfigKeys.KEY_SPARK_DYNAMIC_ALLOCATION_MAX_EXECUTORS(), "inf")); + this.setExecutorMaxNums( + map.getOrDefault(SparkOptions.DYNAMIC_ALLOCATION_MAX_EXECUTORS.key(), "inf")); } else { - this.setExecutorMaxNums(map.get(ConfigKeys.KEY_SPARK_EXECUTOR_INSTANCES())); + this.setExecutorMaxNums(map.get(SparkOptions.EXECUTOR_INSTANCES.key())); } } @@ -362,8 +367,8 @@ public ApplicationType getApplicationType() { return ApplicationType.of(appType); } - public SparkK8sPodTemplates getK8sPodTemplates() { - return SparkK8sPodTemplates.of(k8sDriverPodTemplate, k8sExecutorPodTemplate); + public SparkKubernetesPodTemplates getK8sPodTemplates() { + return SparkKubernetesPodTemplates.of(k8sDriverPodTemplate, k8sExecutorPodTemplate); } @JsonIgnore diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplicationBackup.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplicationBackup.java index a9d5933a26..25ef034c09 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplicationBackup.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkApplicationBackup.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.core.entity; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.Workspace; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; @@ -70,13 +70,13 @@ private void renderPath(SparkApplication application) { case LOCAL: this.path = String.format( "%s/%d/%d", - Workspace.local().APP_BACKUPS(), application.getId(), createTime.getTime()); + Workspace.LOCAL.backups, application.getId(), createTime.getTime()); break; case YARN_CLUSTER: case YARN_CLIENT: this.path = String.format( "%s/%d/%d", - Workspace.remote().APP_BACKUPS(), application.getId(), createTime.getTime()); + Workspace.REMOTE.backups, application.getId(), createTime.getTime()); break; default: throw new UnsupportedOperationException( diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkEnv.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkEnv.java index c153f245a1..3717218e55 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkEnv.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/SparkEnv.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.core.entity; -import org.apache.streampark.common.conf.SparkVersion; +import org.apache.streampark.common.core.SparkVersion; import org.apache.streampark.common.util.DeflaterUtils; import org.apache.streampark.console.base.exception.ApiDetailException; diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Variable.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Variable.java index 1a20768d86..6a2f233e37 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Variable.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Variable.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.core.entity; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; import org.apache.streampark.console.base.mybatis.entity.BaseEntity; import com.baomidou.mybatisplus.annotation.IdType; diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java index 0a3062f174..d009032ef1 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java @@ -17,15 +17,11 @@ package org.apache.streampark.console.core.runner; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.conf.InternalConfigHolder; -import org.apache.streampark.common.conf.InternalOption; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.enums.StorageType; import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.common.util.AssertUtils; -import org.apache.streampark.common.util.SystemPropertyUtils; +import org.apache.streampark.console.base.config.SpringConfigurationInitializer; import org.apache.streampark.console.base.util.WebUtils; import org.apache.streampark.console.core.entity.FlinkEnv; import org.apache.streampark.console.core.entity.SparkEnv; @@ -35,12 +31,10 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; import org.springframework.context.ApplicationContext; import org.springframework.core.annotation.Order; -import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; import java.io.File; @@ -55,34 +49,51 @@ import static org.apache.streampark.common.enums.StorageType.LFS; +/** + * Prepares Console runtime resources after the Spring application context has started. + * + *

Configuration resolution is delegated to {@link SpringConfigurationInitializer}. This runner + * coordinates only the dependent startup work: applying database-backed Maven settings and + * preparing local or remote artifact storage. + */ @Order(1) @Slf4j @Component public class EnvInitializer implements ApplicationRunner { - @Autowired - private ApplicationContext context; - - @Autowired - private SettingService settingService; + private final ApplicationContext context; + private final SettingService settingService; + private final SpringConfigurationInitializer springConfigurationInitializer; private final Set initialized = new HashSet<>(2); private final FileFilter fileFilter = p -> !".gitkeep".equals(p.getName()); private static final Pattern PATTERN_FLINK_SHIMS_JAR = Pattern.compile( - "^streampark-flink-shims_flink-(1\\.1[7-9]|1\\.2[0-9]|2\\.[0-3])-(.*).jar$", + "^streampark-flink-shims_flink-(1\\.1[8-9]|1\\.2[0-9]|2\\.[0-3])-(.*).jar$", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); - private static final Pattern PATTERN_FLINK_SHIMS_BASE_JAR = Pattern.compile( - "^streampark-flink-shims-base(-v2)?-(.*)\\.jar$", - Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + /** + * Creates the ordered Console runtime initializer. + * + * @param context active Spring application context + * @param settingService persistent Console setting service + * @param springConfigurationInitializer Spring-to-StreamPark configuration bridge + */ + public EnvInitializer( + ApplicationContext context, + SettingService settingService, + SpringConfigurationInitializer springConfigurationInitializer) { + this.context = context; + this.settingService = settingService; + this.springConfigurationInitializer = springConfigurationInitializer; + } @SneakyThrows @Override public void run(ApplicationArguments args) throws Exception { - // init InternalConfig + // Resolve framework-specific sources after Spring has applied environment and CLI rules. initConfig(); boolean isTest = Arrays.asList(context.getEnvironment().getActiveProfiles()).contains("test"); @@ -93,31 +104,9 @@ public void run(ApplicationArguments args) throws Exception { } private void initConfig() { - Environment env = context.getEnvironment(); - InternalConfigHolder.initConfigHub(); - // override config from spring application.yaml - InternalConfigHolder.keys().stream() - .filter(env::containsProperty) - .forEach( - key -> { - InternalOption config = InternalConfigHolder.getConfig(key); - AssertUtils.notNull(config); - InternalConfigHolder.set(config, env.getProperty(key, config.classType())); - }); - - InternalConfigHolder.log(); + springConfigurationInitializer.initialize(context.getEnvironment()); settingService.getMavenConfig().updateConfig(); - - // overwrite system variable HADOOP_USER_NAME - String hadoopUserName = InternalConfigHolder.get(CommonConfig.STREAMPARK_HADOOP_USER_NAME()); - overrideSystemProp(ConfigKeys.KEY_HADOOP_USER_NAME(), hadoopUserName); - } - - private void overrideSystemProp(String key, String defaultValue) { - String value = context.getEnvironment().getProperty(key, defaultValue); - log.info("initialize system properties: key:{}, value:{}", key, value); - SystemPropertyUtils.set(key, value); } public synchronized void storageInitialize(StorageType storageType) { @@ -127,7 +116,7 @@ public synchronized void storageInitialize(StorageType storageType) { } FsOperator fsOperator = FsOperator.of(storageType); - Workspace workspace = Workspace.of(storageType); + Workspace workspace = LFS == storageType ? Workspace.LOCAL : Workspace.REMOTE; // 1. prepare workspace dir prepareWorkspace(storageType, fsOperator, workspace); @@ -146,35 +135,36 @@ public synchronized void storageInitialize(StorageType storageType) { private void prepareWorkspace( StorageType storageType, FsOperator fsOperator, Workspace workspace) { if (LFS == storageType) { - fsOperator.mkdirsIfNotExists(Workspace.APP_LOCAL_DIST()); + fsOperator.mkdirsIfNotExists(Workspace.APP_LOCAL_DIST); } Arrays.asList( - workspace.APP_UPLOADS(), - workspace.APP_WORKSPACE(), - workspace.APP_BACKUPS(), - workspace.APP_SAVEPOINTS(), - workspace.APP_PYTHON(), - workspace.APP_JARS()) + workspace.uploads, + workspace.workspace, + workspace.backups, + workspace.savepoints, + workspace.python, + workspace.jars) .forEach(fsOperator::mkdirsIfNotExists); } private static void createMvnLocalRepoDir() { - String localMavenRepo = Workspace.MAVEN_LOCAL_PATH(); - if (FsOperator.lfs().exists(localMavenRepo)) { + String localMavenRepo = Workspace.MAVEN_LOCAL_PATH; + if (!FsOperator.lfs().exists(localMavenRepo)) { FsOperator.lfs().mkdirs(localMavenRepo); } } private void uploadClientJar(Workspace workspace, FsOperator fsOperator) { File client = WebUtils.getAppClientDir(); + File[] clientFiles = client.listFiles(fileFilter); AssertUtils.required( - client.exists() && client.listFiles().length > 0, + client.isDirectory() && clientFiles != null && clientFiles.length > 0, client.getAbsolutePath().concat(" is not exists or empty directory ")); - String appClient = workspace.APP_CLIENT(); + String appClient = workspace.client; fsOperator.mkCleanDirs(appClient); - for (File file : client.listFiles(fileFilter)) { + for (File file : clientFiles) { log.info("load client:{} to {}", file.getName(), appClient); fsOperator.upload(file.getAbsolutePath(), appClient); } @@ -185,13 +175,9 @@ private void uploadShimsJar(Workspace workspace, FsOperator fsOperator) { .listFiles(pathname -> pathname.getName().matches(PATTERN_FLINK_SHIMS_JAR.pattern())); AssertUtils.required(shims != null && shims.length > 0, "streampark-flink-shims jar not exist"); - String appShims = workspace.APP_SHIMS(); + String appShims = workspace.shims; fsOperator.delete(appShims); - File[] shimsBaseJars = - WebUtils.getAppLibDir().listFiles( - pathname -> pathname.getName().matches(PATTERN_FLINK_SHIMS_BASE_JAR.pattern())); - for (File file : shims) { Matcher matcher = PATTERN_FLINK_SHIMS_JAR.matcher(file.getName()); if (matcher.matches()) { @@ -200,34 +186,18 @@ private void uploadShimsJar(Workspace workspace, FsOperator fsOperator) { fsOperator.mkdirs(shimsPath); log.info("load shims:{} to {}", file.getName(), shimsPath); fsOperator.upload(file.getAbsolutePath(), shimsPath); - if (isFlink2MajorVersion(version) && shimsBaseJars != null) { - uploadShimsBaseJars(fsOperator, shimsBaseJars, shimsPath); - } } } } - private static boolean isFlink2MajorVersion(String majorVersion) { - int dot = majorVersion.indexOf('.'); - String major = dot < 0 ? majorVersion : majorVersion.substring(0, dot); - return "2".equals(major); - } - - private void uploadShimsBaseJars(FsOperator fsOperator, File[] shimsBaseJars, String shimsPath) { - for (File baseJar : shimsBaseJars) { - log.info("load shims base:{} to {}", baseJar.getName(), shimsPath); - fsOperator.upload(baseJar.getAbsolutePath(), shimsPath); - } - } - public void checkFlinkEnv(StorageType storageType, FlinkEnv flinkEnv) throws IOException { String flinkLocalHome = flinkEnv.getFlinkHome(); if (StringUtils.isBlank(flinkLocalHome)) { throw new ExceptionInInitializerError( "[StreamPark] FLINK_HOME is undefined,Make sure that Flink is installed."); } - Workspace workspace = Workspace.of(storageType); - String appFlink = workspace.APP_FLINK(); + Workspace workspace = LFS == storageType ? Workspace.LOCAL : Workspace.REMOTE; + String appFlink = workspace.flink; FsOperator fsOperator = FsOperator.of(storageType); if (!fsOperator.exists(appFlink)) { log.info("checkFlinkEnv, now mkdir [{}] starting ...", appFlink); @@ -251,8 +221,8 @@ public void checkSparkEnv(StorageType storageType, SparkEnv sparkEnv) throws IOE throw new ExceptionInInitializerError( "[StreamPark] SPARK_HOME is undefined,Make sure that Spark is installed."); } - Workspace workspace = Workspace.of(storageType); - String appSpark = workspace.APP_SPARK(); + Workspace workspace = LFS == storageType ? Workspace.LOCAL : Workspace.REMOTE; + String appSpark = workspace.spark; FsOperator fsOperator = FsOperator.of(storageType); if (!fsOperator.exists(appSpark)) { log.info("checkSparkEnv, now mkdir [{}] starting ...", appSpark); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/QuickStartRunner.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/QuickStartRunner.java index 0b3aab314c..4444b315e0 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/QuickStartRunner.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/QuickStartRunner.java @@ -17,9 +17,9 @@ package org.apache.streampark.console.core.runner; +import org.apache.streampark.common.configuration.GroupedOptionsParser; import org.apache.streampark.common.enums.ClusterState; import org.apache.streampark.common.enums.FlinkDeployMode; -import org.apache.streampark.common.util.FlinkConfigurationUtils; import org.apache.streampark.console.core.entity.FlinkApplication; import org.apache.streampark.console.core.entity.FlinkCluster; import org.apache.streampark.console.core.entity.FlinkEnv; @@ -65,7 +65,7 @@ public class QuickStartRunner implements ApplicationRunner { public void run(ApplicationArguments args) throws Exception { Map> map = - FlinkConfigurationUtils.extractMultipleArgumentsAsJava(args.getSourceArgs()); + GroupedOptionsParser.parse(args.getSourceArgs()); Map quickstart = map.get("quickstart"); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProxyService.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProxyService.java index 8722db5a63..fca9bfe240 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProxyService.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProxyService.java @@ -21,19 +21,18 @@ import org.apache.streampark.console.core.entity.FlinkApplication; import org.apache.streampark.console.core.entity.SparkApplication; -import org.springframework.http.ResponseEntity; - import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; public interface ProxyService { - ResponseEntity proxyFlink(HttpServletRequest request, FlinkApplication app) throws Exception; + void proxyFlink(HttpServletRequest request, HttpServletResponse response, FlinkApplication app) throws Exception; - ResponseEntity proxySpark(HttpServletRequest request, SparkApplication app) throws Exception; + void proxySpark(HttpServletRequest request, HttpServletResponse response, SparkApplication app) throws Exception; - ResponseEntity proxyYarn(HttpServletRequest request, ApplicationLog log) throws Exception; + void proxyYarn(HttpServletRequest request, HttpServletResponse response, ApplicationLog log) throws Exception; - ResponseEntity proxyHistory(HttpServletRequest request, ApplicationLog log) throws Exception; + void proxyHistory(HttpServletRequest request, HttpServletResponse response, ApplicationLog log) throws Exception; - ResponseEntity proxyFlinkCluster(HttpServletRequest request, Long clusterId) throws Exception; + void proxyFlinkCluster(HttpServletRequest request, HttpServletResponse response, Long clusterId) throws Exception; } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java index dcec680afe..414db6c7a4 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java @@ -17,21 +17,23 @@ package org.apache.streampark.console.core.service.application.impl; -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.conf.Workspace; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.JvmOptionsParser; +import org.apache.streampark.common.configuration.Workspace; +import org.apache.streampark.common.configuration.option.ApplicationOptions; +import org.apache.streampark.common.configuration.option.DeploymentOptions; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.ClusterState; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; -import org.apache.streampark.common.enums.FlinkK8sRestExposedType; +import org.apache.streampark.common.enums.FlinkKubernetesRestExposedType; import org.apache.streampark.common.enums.FlinkRestoreMode; import org.apache.streampark.common.enums.ResolveOrder; import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.DeflaterUtils; import org.apache.streampark.common.util.ExceptionUtils; -import org.apache.streampark.common.util.FlinkConfigurationUtils; import org.apache.streampark.common.util.HadoopUtils; import org.apache.streampark.console.base.exception.ApiAlertException; import org.apache.streampark.console.base.exception.ApplicationException; @@ -68,20 +70,20 @@ import org.apache.streampark.console.core.service.application.FlinkApplicationConfigService; import org.apache.streampark.console.core.service.application.FlinkApplicationInfoService; import org.apache.streampark.console.core.service.application.FlinkApplicationManageService; +import org.apache.streampark.console.core.util.FlinkEnvUtils; import org.apache.streampark.console.core.util.ServiceHelper; import org.apache.streampark.console.core.watcher.FlinkAppHttpWatcher; import org.apache.streampark.console.core.watcher.FlinkClusterWatcher; import org.apache.streampark.console.core.watcher.FlinkK8sWatcherWrapper; import org.apache.streampark.flink.client.FlinkClient; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.JobClientTarget; -import org.apache.streampark.flink.client.bean.SavepointCancelOptions; -import org.apache.streampark.flink.client.bean.SubmitApplicationSpec; -import org.apache.streampark.flink.client.bean.SubmitClusterSpec; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.kubernetes.FlinkK8sWatcher; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.JobClientTarget; +import org.apache.streampark.flink.client.request.SubmitApplicationSpec; +import org.apache.streampark.flink.client.request.SubmitClusterSpec; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.SubmitResponse; +import org.apache.streampark.flink.kubernetes.FlinkKubernetesWatcher; import org.apache.streampark.flink.kubernetes.helper.KubernetesDeploymentHelper; import org.apache.streampark.flink.kubernetes.ingress.IngressController; import org.apache.streampark.flink.kubernetes.model.TrackId; @@ -94,6 +96,7 @@ import org.apache.flink.configuration.JobManagerOptions; import org.apache.flink.configuration.MemorySize; import org.apache.flink.configuration.RestOptions; +import org.apache.flink.configuration.TaskManagerOptions; import org.apache.flink.kubernetes.shaded.io.fabric8.kubernetes.client.KubernetesClientException; import org.apache.flink.runtime.jobgraph.SavepointConfigOptions; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -168,7 +171,7 @@ public class FlinkApplicationActionServiceImpl private SettingService settingService; @Autowired - private FlinkK8sWatcher k8SFlinkTrackMonitor; + private FlinkKubernetesWatcher k8SFlinkTrackMonitor; @Autowired private FlinkApplicationBuildPipelineService appBuildPipeService; @@ -209,7 +212,7 @@ public void revoke(Long appId) throws ApplicationException { // 3) restore related status LambdaUpdateWrapper updateWrapper = Wrappers.lambdaUpdate(); updateWrapper.eq(FlinkApplication::getId, application.getId()); - if (application.isFlinkSql()) { + if (application.isFlinkSqlJob()) { updateWrapper.set(FlinkApplication::getRelease, ReleaseStateEnum.FAILED.get()); } else { updateWrapper.set(FlinkApplication::getRelease, ReleaseStateEnum.NEED_RELEASE.get()); @@ -302,7 +305,7 @@ public void cancel(FlinkApplication appParam) throws Exception { properties.put(RestOptions.PORT.key(), activeAddress.getPort()); } - Tuple3 clusterIdNamespace = + Tuple3 clusterIdNamespace = getNamespaceClusterId(application); String namespace = clusterIdNamespace.t1; String clusterId = clusterIdNamespace.t2; @@ -310,15 +313,14 @@ public void cancel(FlinkApplication appParam) throws Exception { CancelRequest cancelRequest = new CancelRequest( application.getId(), - flinkEnv.getFlinkVersion(), + FlinkEnvUtils.version(flinkEnv), FlinkDeployMode.of(application.getDeployMode()), properties, new JobClientTarget(clusterId, application.getJobId(), namespace), - new SavepointCancelOptions( - Boolean.TRUE.equals(appParam.getRestoreOrTriggerSavepoint()), - Boolean.TRUE.equals(appParam.getDrain()), - customSavepoint, - Boolean.TRUE.equals(appParam.getNativeFormat()))); + Boolean.TRUE.equals(appParam.getRestoreOrTriggerSavepoint()), + Boolean.TRUE.equals(appParam.getDrain()), + customSavepoint, + Boolean.TRUE.equals(appParam.getNativeFormat())); final Date triggerTime = new Date(); CompletableFuture cancelFuture = @@ -424,12 +426,12 @@ public void start(FlinkApplication appParam, boolean auto) throws Exception { applicationManageService.toEffective(application); Map extraParameter = new HashMap<>(0); - if (application.isFlinkSql()) { + if (application.isFlinkSqlJob()) { FlinkSql flinkSql = flinkSqlService.getEffective(application.getId(), true); // Get the sql of the replaced placeholder String realSql = variableService.replaceVariable(application.getTeamId(), flinkSql.getSql()); flinkSql.setSql(DeflaterUtils.zipString(realSql)); - extraParameter.put(ConfigKeys.KEY_FLINK_SQL(null), flinkSql.getSql()); + extraParameter.put(ApplicationOptions.SQL.key(), flinkSql.getSql()); } Tuple2 userJarAndAppConf = getUserJarAndAppConf(flinkEnv, application); @@ -444,11 +446,11 @@ public void start(FlinkApplication appParam, boolean auto) throws Exception { StringUtils.isBlank(appParam.getArgs()) ? application.getArgs() : appParam.getArgs(); String applicationArgs = variableService.replaceVariable(application.getTeamId(), args); - Tuple3 clusterIdNamespace = + Tuple3 clusterIdNamespace = getNamespaceClusterId(application); String k8sNamespace = clusterIdNamespace.t1; String k8sClusterId = clusterIdNamespace.t2; - FlinkK8sRestExposedType exposedType = clusterIdNamespace.t3; + FlinkKubernetesRestExposedType exposedType = clusterIdNamespace.t3; String dynamicProperties = StringUtils.isBlank(appParam.getDynamicProperties()) @@ -457,11 +459,12 @@ public void start(FlinkApplication appParam, boolean auto) throws Exception { SubmitRequest submitRequest = new SubmitRequest( - flinkEnv.getFlinkVersion(), + FlinkEnvUtils.version(flinkEnv), FlinkDeployMode.of(application.getDeployMode()), getProperties(application, dynamicProperties), SubmitApplicationSpec.builder() - .flinkYaml(flinkEnv.getFlinkConf()) + .flinkYaml(FlinkEnvUtils.compressedYaml(flinkEnv)) + .standardYaml(FlinkEnvUtils.isStandardYaml(flinkEnv)) .jobType(FlinkJobType.of(application.getJobType())) .id(application.getId()) .jobId(new JobID().toHexString()) @@ -517,11 +520,11 @@ private void processForSuccess( FlinkApplication flinkApplication) { applicationLog.setSuccess(true); if (response.flinkConfig() != null) { - String jmMemory = response.flinkConfig().get(ConfigKeys.KEY_FLINK_JM_PROCESS_MEMORY()); + String jmMemory = response.flinkConfig().get(JobManagerOptions.TOTAL_PROCESS_MEMORY.key()); if (jmMemory != null) { flinkApplication.setJmMemory(MemorySize.parse(jmMemory).getMebiBytes()); } - String tmMemory = response.flinkConfig().get(ConfigKeys.KEY_FLINK_TM_PROCESS_MEMORY()); + String tmMemory = response.flinkConfig().get(TaskManagerOptions.TOTAL_PROCESS_MEMORY.key()); if (tmMemory != null) { flinkApplication.setTmMemory(MemorySize.parse(tmMemory).getMebiBytes()); } @@ -694,7 +697,7 @@ private Tuple2 getUserJarAndAppConf( : String.format("yaml://%s", applicationConfig.getContent()); // 3) client if (FlinkDeployMode.YARN_APPLICATION == deployModeEnum) { - String clientPath = Workspace.remote().APP_CLIENT(); + String clientPath = Workspace.REMOTE.client; flinkUserJar = String.format("%s/%s", clientPath, sqlDistJar); } break; @@ -721,7 +724,7 @@ private Tuple2 getUserJarAndAppConf( appConf = String.format( "json://{\"%s\":\"%s\"}", - ConfigKeys.KEY_FLINK_APPLICATION_MAIN_CLASS(), application.getMainClass()); + ApplicationOptions.MAIN_CLASS.key(), application.getMainClass()); } else { switch (application.getApplicationType()) { case STREAMPARK_FLINK: @@ -739,7 +742,7 @@ private Tuple2 getUserJarAndAppConf( appConf = String.format( "json://{\"%s\":\"%s\"}", - ConfigKeys.KEY_FLINK_APPLICATION_MAIN_CLASS(), application.getMainClass()); + ApplicationOptions.MAIN_CLASS.key(), application.getMainClass()); break; default: throw new IllegalArgumentException( @@ -803,23 +806,24 @@ private Map getProperties( "The yarn session clusterId=%s cannot be find, maybe the clusterId is wrong or " + "the cluster has been deleted. Please contact the Admin.", application.getFlinkClusterId())); - properties.put(ConfigKeys.KEY_YARN_APP_ID(), cluster.getClusterId()); + properties.put(DeploymentOptions.YARN_APPLICATION_ID.key(), cluster.getClusterId()); } else { String yarnQueue = - (String) application.getHotParamsMap().get(ConfigKeys.KEY_YARN_APP_QUEUE()); + (String) application.getHotParamsMap().get(DeploymentOptions.YARN_QUEUE.key()); String yarnLabelExpr = - (String) application.getHotParamsMap().get(ConfigKeys.KEY_YARN_APP_NODE_LABEL()); + (String) application.getHotParamsMap().get(DeploymentOptions.YARN_NODE_LABEL.key()); Optional.ofNullable(yarnQueue) - .ifPresent(yq -> properties.put(ConfigKeys.KEY_YARN_APP_QUEUE(), yq)); + .ifPresent(yq -> properties.put(DeploymentOptions.YARN_QUEUE.key(), yq)); Optional.ofNullable(yarnLabelExpr) - .ifPresent(yLabel -> properties.put(ConfigKeys.KEY_YARN_APP_NODE_LABEL(), yLabel)); + .ifPresent( + yLabel -> properties.put(DeploymentOptions.YARN_NODE_LABEL.key(), yLabel)); } } else if (FlinkDeployMode.isKubernetesMode(application.getDeployModeEnum())) { - properties.put(ConfigKeys.KEY_K8S_IMAGE_PULL_POLICY(), "Always"); + properties.put(FlinkOptions.KUBERNETES_IMAGE_PULL_POLICY.key(), "Always"); } if (FlinkDeployMode.isKubernetesApplicationMode(application.getDeployMode())) { - properties.put(JobManagerOptions.ARCHIVE_DIR.key(), Workspace.ARCHIVES_FILE_PATH()); + properties.put(JobManagerOptions.ARCHIVE_DIR.key(), Workspace.ARCHIVES_FILE_PATH); } if (application.getAllowNonRestored()) { @@ -827,7 +831,7 @@ private Map getProperties( } Map dynamicProperties = - FlinkConfigurationUtils.extractDynamicPropertiesAsJava(runtimeProperties); + JvmOptionsParser.parse(runtimeProperties).toMap(); properties.putAll(dynamicProperties); ResolveOrder resolveOrder = ResolveOrder.of(application.getResolveOrder()); if (resolveOrder != null) { @@ -896,11 +900,11 @@ private void checkBeforeStart(FlinkApplication application) { "[StreamPark] The flink cluster not running, please start it"); } - private Tuple3 getNamespaceClusterId( - FlinkApplication application) { + private Tuple3 getNamespaceClusterId( + FlinkApplication application) { String clusterId = null; String k8sNamespace = null; - FlinkK8sRestExposedType exposedType = null; + FlinkKubernetesRestExposedType exposedType = null; switch (application.getDeployModeEnum()) { case YARN_APPLICATION: case YARN_PER_JOB: diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBackupServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBackupServiceImpl.java index 9a84d354fa..cdcdf8368a 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBackupServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBackupServiceImpl.java @@ -89,7 +89,7 @@ public void rollback(FlinkApplicationBackup bakParam) { // If necessary, perform the backup first if (bakParam.isBackup()) { application.setBackUpDescription(bakParam.getDescription()); - if (application.isFlinkSql()) { + if (application.isFlinkSqlJob()) { FlinkSql flinkSql = flinkSqlService.getEffective(application.getId(), false); backup(application, flinkSql); } else { @@ -107,7 +107,7 @@ public void rollback(FlinkApplicationBackup bakParam) { effectiveService.saveOrUpdate( bakParam.getAppId(), EffectiveTypeEnum.CONFIG, bakParam.getId()); // if flink sql task, will be rollback sql and dependencies - if (application.isFlinkSql()) { + if (application.isFlinkSqlJob()) { effectiveService.saveOrUpdate( bakParam.getAppId(), EffectiveTypeEnum.FLINKSQL, bakParam.getSqlId()); } @@ -141,7 +141,7 @@ public void revoke(FlinkApplication appParam) { if (!backUpPages.getRecords().isEmpty()) { FlinkApplicationBackup backup = backUpPages.getRecords().get(0); String path = backup.getPath(); - appParam.getFsOperator().move(path, appParam.getWorkspace().APP_WORKSPACE()); + appParam.getFsOperator().move(path, appParam.getWorkspace().workspace); super.removeById(backup.getId()); } } @@ -154,10 +154,9 @@ public void remove(FlinkApplication appParam) { .getFsOperator() .delete( appParam - .getWorkspace() - .APP_BACKUPS() - .concat("/") - .concat(appParam.getId().toString())); + .getWorkspace().backups + .concat("/") + .concat(appParam.getId().toString())); } catch (Exception e) { log.error(e.getMessage(), e); } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBuildPipelineServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBuildPipelineServiceImpl.java index 45ecd0041e..07fdf8d81f 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBuildPipelineServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationBuildPipelineServiceImpl.java @@ -17,8 +17,9 @@ package org.apache.streampark.console.core.service.application.impl; -import org.apache.streampark.common.conf.Workspace; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.Workspace; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; import org.apache.streampark.common.util.AssertUtils; @@ -48,6 +49,8 @@ import org.apache.streampark.console.core.service.application.FlinkApplicationInfoService; import org.apache.streampark.console.core.service.application.FlinkApplicationManageService; import org.apache.streampark.console.core.util.ApplicationBuildPipelineUtils; +import org.apache.streampark.console.core.util.FlinkApplicationConfigUtils; +import org.apache.streampark.console.core.util.FlinkEnvUtils; import org.apache.streampark.console.core.util.ServiceHelper; import org.apache.streampark.console.core.watcher.FlinkAppHttpWatcher; import org.apache.streampark.flink.packer.docker.DockerConf; @@ -169,7 +172,7 @@ public boolean buildApplication(@Nonnull Long appId, boolean forceBuild) { return true; } // rollback - if (app.isNeedRollback() && app.isFlinkSql()) { + if (app.isNeedRollback() && app.isFlinkSqlJob()) { flinkSqlService.rollback(app); } @@ -207,7 +210,7 @@ public void onStart(PipelineSnapshot snapshot) { applicationInfoService.checkEnv(app); - String appUploads = app.getWorkspace().APP_UPLOADS(); + String appUploads = app.getWorkspace().uploads; ApplicationBuildPipelineUtils.prepareBuildResources( app.isFlinkJarOrPyFlink(), () -> ApplicationBuildPipelineUtils.prepareJarJobHome( @@ -238,20 +241,20 @@ public void onFinish(PipelineSnapshot snapshot, BuildResult result) { ApplicationBuildPipelineUtils.applySuccessfulRelease( app, () -> { - if (app.isFlinkSql()) { + if (app.isFlinkSqlJob()) { applicationManageService.toEffective(app); } else if (app.isStreamParkType()) { FlinkApplicationConfig config = applicationConfigService.getLatest(app.getId()); if (config != null) { - config.setToApplication(app); + FlinkApplicationConfigUtils.applyTo(config, app); applicationConfigService.toEffective( app.getId(), app.getConfigId()); } } }); if (!app.isNeedRollback()) { - if (app.isFlinkSql() && newFlinkSql != null) { + if (app.isFlinkSqlJob() && newFlinkSql != null) { backUpService.backup(app, newFlinkSql); } else { backUpService.backup(app, null); @@ -328,9 +331,10 @@ private void checkBuildEnv(Long appId, boolean forceBuild) { String checkEnvErrorMessage = "Check flink env failed, please check the flink version of this job"; FlinkEnv env = flinkEnvService.getByIdOrDefault(app.getVersionId()); ApiAlertException.throwIfNull(env, checkEnvErrorMessage); - boolean checkVersion = env.getFlinkVersion().checkVersion(false); + FlinkVersion flinkVersion = FlinkEnvUtils.version(env); + boolean checkVersion = flinkVersion.checkVersion(false); ApiAlertException.throwIfFalse( - checkVersion, "Unsupported flink version:" + env.getFlinkVersion().version()); + checkVersion, "Unsupported flink version:" + flinkVersion.version()); // 2) check env boolean envOk = applicationInfoService.checkEnv(app); @@ -424,7 +428,7 @@ private FlinkK8sApplicationBuildRequest buildFlinkK8sApplicationBuildRequest( flinkUserJar, app.getDeployModeEnum(), app.getJobTypeEnum(), - flinkEnv.getFlinkVersion(), + FlinkEnvUtils.version(flinkEnv), getMergedDependencyInfo(app), app.getJobName(), app.getK8sNamespace(), @@ -452,7 +456,7 @@ private FlinkK8sSessionBuildRequest buildFlinkK8sSessionBuildRequest( flinkUserJar, app.getDeployModeEnum(), app.getJobTypeEnum(), - flinkEnv.getFlinkVersion(), + FlinkEnvUtils.version(flinkEnv), getMergedDependencyInfo(app), app.getClusterId(), app.getK8sNamespace()); @@ -472,7 +476,7 @@ private FlinkRemotePerJobBuildRequest buildFlinkRemotePerJobBuildRequest( app.isFlinkJar(), app.getDeployModeEnum(), app.getJobTypeEnum(), - flinkEnv.getFlinkVersion(), + FlinkEnvUtils.version(flinkEnv), getMergedDependencyInfo(app)); } @@ -498,10 +502,10 @@ private String retrieveFlinkUserJar(FlinkEnv flinkEnv, FlinkApplication app) { case FLINK_SQL: String sqlDistJar = ServiceHelper.getFlinkSqlClientJar(flinkEnv); if (app.getDeployModeEnum() == FlinkDeployMode.YARN_APPLICATION) { - String clientPath = Workspace.remote().APP_CLIENT(); + String clientPath = Workspace.REMOTE.client; return String.format("%s/%s", clientPath, sqlDistJar); } - return Workspace.local().APP_CLIENT().concat("/").concat(sqlDistJar); + return Workspace.LOCAL.client.concat("/").concat(sqlDistJar); default: throw new UnsupportedOperationException( "[StreamPark] unsupported JobType: " + app.getJobTypeEnum()); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationConfigServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationConfigServiceImpl.java index 21d84e5f42..4801a3af99 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationConfigServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationConfigServiceImpl.java @@ -99,7 +99,7 @@ public void setLatest(Long appId, Long configId) { public synchronized void update(FlinkApplication appParam, Boolean latest) { // flink sql job FlinkApplicationConfig latestConfig = getLatest(appParam.getId()); - if (appParam.isFlinkSql()) { + if (appParam.isFlinkSqlJob()) { updateForFlinkSqlJob(appParam, latest, latestConfig); } else { updateForNonFlinkSqlJob(appParam, latest, latestConfig); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java index 153498dccc..3ecadd9a15 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java @@ -40,11 +40,11 @@ import org.apache.streampark.console.core.service.FlinkEnvService; import org.apache.streampark.console.core.service.SavepointService; import org.apache.streampark.console.core.service.application.FlinkApplicationInfoService; +import org.apache.streampark.console.core.util.FlinkShellCommandBuilder; import org.apache.streampark.console.core.watcher.FlinkAppHttpWatcher; import org.apache.streampark.console.core.watcher.FlinkClusterWatcher; import org.apache.streampark.console.core.watcher.FlinkK8sWatcherWrapper; -import org.apache.streampark.flink.core.conf.ParameterCli; -import org.apache.streampark.flink.kubernetes.FlinkK8sWatcher; +import org.apache.streampark.flink.kubernetes.FlinkKubernetesWatcher; import org.apache.streampark.flink.kubernetes.helper.KubernetesDeploymentHelper; import org.apache.streampark.flink.kubernetes.model.FlinkMetricCV; @@ -100,7 +100,7 @@ public class FlinkApplicationInfoServiceImpl extends ServiceImpl userJarAndAppConf = getUserJarAndAppConf(sparkEnv, application); @@ -303,10 +304,10 @@ public void start(SparkApplication appParam, boolean auto) throws Exception { if (SparkDeployMode.isYarnMode(application.getDeployModeEnum())) { buildResult = new ShadedBuildResponse(null, sparkUserJar, true); if (StringUtils.isNotBlank(application.getYarnQueueName())) { - extraParameter.put(ConfigKeys.KEY_SPARK_YARN_QUEUE_NAME(), application.getYarnQueueName()); + extraParameter.put(SparkOptions.YARN_QUEUE_NAME.key(), application.getYarnQueueName()); } if (StringUtils.isNotBlank(application.getYarnQueueLabel())) { - extraParameter.put(ConfigKeys.KEY_SPARK_YARN_QUEUE_LABEL(), application.getYarnQueueLabel()); + extraParameter.put(SparkOptions.YARN_QUEUE_LABEL.key(), application.getYarnQueueLabel()); } } @@ -432,7 +433,7 @@ private Tuple2 getUserJarAndAppConf( : String.format("yaml://%s", applicationConfig.getContent()); // 3) client if (SparkDeployMode.YARN_CLUSTER == deployModeEnum) { - String clientPath = Workspace.remote().APP_CLIENT(); + String clientPath = Workspace.REMOTE.client; sparkUserJar = String.format("%s/%s", clientPath, sqlDistJar); } break; @@ -476,7 +477,7 @@ private Tuple2 getUserJarAndAppConf( case APACHE_SPARK: appConf = String.format( "json://{\"%s\":\"%s\"}", - ConfigKeys.KEY_FLINK_APPLICATION_MAIN_CLASS(), application.getMainClass()); + ApplicationOptions.MAIN_CLASS.key(), application.getMainClass()); break; default: throw new IllegalArgumentException( diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBackupServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBackupServiceImpl.java index 912d13b4b6..c1d2a452fe 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBackupServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBackupServiceImpl.java @@ -138,7 +138,7 @@ public void revoke(SparkApplication appParam) { if (!backUpPages.getRecords().isEmpty()) { SparkApplicationBackup backup = backUpPages.getRecords().get(0); String path = backup.getPath(); - appParam.getFsOperator().move(path, appParam.getWorkspace().APP_WORKSPACE()); + appParam.getFsOperator().move(path, appParam.getWorkspace().workspace); super.removeById(backup.getId()); } } @@ -151,10 +151,9 @@ public void remove(SparkApplication appParam) { .getFsOperator() .delete( appParam - .getWorkspace() - .APP_BACKUPS() - .concat("/") - .concat(appParam.getId().toString())); + .getWorkspace().backups + .concat("/") + .concat(appParam.getId().toString())); } catch (Exception e) { log.error(e.getMessage(), e); } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBuildPipelineServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBuildPipelineServiceImpl.java index db0268170e..eef3b93f43 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBuildPipelineServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationBuildPipelineServiceImpl.java @@ -17,8 +17,8 @@ package org.apache.streampark.console.core.service.application.impl; -import org.apache.streampark.common.conf.Workspace; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.SparkDeployMode; import org.apache.streampark.common.util.AssertUtils; @@ -175,7 +175,7 @@ public void onStart(PipelineSnapshot snapshot) { applicationInfoService.checkEnv(app); - String appUploads = app.getWorkspace().APP_UPLOADS(); + String appUploads = app.getWorkspace().uploads; ApplicationBuildPipelineUtils.prepareBuildResources( app.isSparkJarOrPySparkJob(), () -> ApplicationBuildPipelineUtils.prepareJarJobHome( @@ -365,10 +365,10 @@ private String retrieveSparkUserJar(SparkEnv sparkEnv, SparkApplication app) { case SPARK_SQL: String sqlDistJar = ServiceHelper.getSparkSqlClientJar(sparkEnv); if (app.getDeployModeEnum() == SparkDeployMode.YARN_CLUSTER) { - String clientPath = Workspace.remote().APP_CLIENT(); + String clientPath = Workspace.REMOTE.client; return String.format("%s/%s", clientPath, sqlDistJar); } - return Workspace.local().APP_CLIENT().concat("/").concat(sqlDistJar); + return Workspace.LOCAL.client.concat("/").concat(sqlDistJar); default: throw new UnsupportedOperationException( "[StreamPark] unsupported JobType: " + app.getJobTypeEnum()); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationInfoServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationInfoServiceImpl.java index b3350c5db6..82d320ff08 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationInfoServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationInfoServiceImpl.java @@ -32,8 +32,8 @@ import org.apache.streampark.console.core.runner.EnvInitializer; import org.apache.streampark.console.core.service.SparkEnvService; import org.apache.streampark.console.core.service.application.SparkApplicationInfoService; +import org.apache.streampark.console.core.util.FlinkShellCommandBuilder; import org.apache.streampark.console.core.watcher.SparkAppHttpWatcher; -import org.apache.streampark.flink.core.conf.ParameterCli; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.yarn.api.records.ApplicationReport; @@ -248,7 +248,7 @@ public String getYarnName(String appConfig) { String[] args = new String[2]; args[0] = "--name"; args[1] = appConfig; - return ParameterCli.read(args); + return FlinkShellCommandBuilder.read(args); } /** diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationManageServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationManageServiceImpl.java index b0cf67c829..87caf53171 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationManageServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationManageServiceImpl.java @@ -17,10 +17,9 @@ package org.apache.streampark.console.core.service.application.impl; -import org.apache.streampark.common.conf.Workspace; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.enums.SparkDeployMode; -import org.apache.streampark.common.enums.StorageType; import org.apache.streampark.common.fs.HdfsOperator; import org.apache.streampark.common.util.DeflaterUtils; import org.apache.streampark.console.base.domain.RestRequest; @@ -193,9 +192,9 @@ private void removeApp(SparkApplication application) { try { application .getFsOperator() - .delete(application.getWorkspace().APP_WORKSPACE().concat("/").concat(appId.toString())); + .delete(application.getWorkspace().workspace.concat("/").concat(appId.toString())); // try to delete yarn-application, and leave no trouble. - String path = Workspace.of(StorageType.HDFS).APP_WORKSPACE().concat("/").concat(appId.toString()); + String path = Workspace.REMOTE.workspace.concat("/").concat(appId.toString()); if (HdfsOperator.exists(path)) { HdfsOperator.delete(path); } @@ -266,7 +265,7 @@ public boolean create(SparkApplication appParam) { } if (appParam.isFromUploadJob()) { String jarPath = String.format( - "%s/%d/%s", Workspace.local().APP_UPLOADS(), appParam.getTeamId(), appParam.getJar()); + "%s/%d/%s", Workspace.LOCAL.uploads, appParam.getTeamId(), appParam.getJar()); if (!new File(jarPath).exists()) { Resource resource = resourceService.findByResourceName(appParam.getTeamId(), appParam.getJar()); if (resource != null && StringUtils.isNotBlank(resource.getFilePath())) { diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkClusterServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkClusterServiceImpl.java index fbcba85624..eeaa3790cd 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkClusterServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkClusterServiceImpl.java @@ -31,13 +31,14 @@ import org.apache.streampark.console.core.service.FlinkEnvService; import org.apache.streampark.console.core.service.YarnQueueService; import org.apache.streampark.console.core.service.application.FlinkApplicationInfoService; +import org.apache.streampark.console.core.util.FlinkEnvUtils; import org.apache.streampark.console.core.watcher.FlinkClusterWatcher; import org.apache.streampark.flink.client.FlinkClient; -import org.apache.streampark.flink.client.bean.DeployRequest; -import org.apache.streampark.flink.client.bean.DeployResponse; -import org.apache.streampark.flink.client.bean.KubernetesDeployParam; -import org.apache.streampark.flink.client.bean.ShutDownRequest; -import org.apache.streampark.flink.client.bean.ShutDownResponse; +import org.apache.streampark.flink.client.request.DeployRequest; +import org.apache.streampark.flink.client.request.KubernetesDeploySpec; +import org.apache.streampark.flink.client.request.ShutdownRequest; +import org.apache.streampark.flink.client.response.DeployResponse; +import org.apache.streampark.flink.client.response.ShutdownResponse; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -246,8 +247,8 @@ public void shutdown(FlinkCluster cluster) { FlinkCluster flinkCluster = this.getById(cluster.getId()); try { - ShutDownResponse shutDownResponse = shutdownInternal(flinkCluster, flinkCluster.getClusterId()); - ApiAlertException.throwIfNull(shutDownResponse, "Get shutdown response failed"); + ShutdownResponse shutdownResponse = shutdownInternal(flinkCluster, flinkCluster.getClusterId()); + ApiAlertException.throwIfNull(shutdownResponse, "Get shutdown response failed"); flinkCluster.setClusterState(ClusterState.CANCELED.getState()); flinkCluster.setEndTime(new Date()); updateById(flinkCluster); @@ -410,27 +411,27 @@ private boolean isYarnNotDefaultQueue(FlinkCluster cluster) { && !yarnQueueService.isDefaultQueue(cluster.getYarnQueue()); } - private ShutDownResponse shutdownInternal(FlinkCluster flinkCluster, + private ShutdownResponse shutdownInternal(FlinkCluster flinkCluster, String clusterId) throws InterruptedException, ExecutionException, TimeoutException { - ShutDownRequest stopRequest = new ShutDownRequest( - flinkEnvService.getById(flinkCluster.getVersionId()).getFlinkVersion(), + ShutdownRequest stopRequest = new ShutdownRequest( + FlinkEnvUtils.version(flinkEnvService.getById(flinkCluster.getVersionId())), flinkCluster.getFlinkDeployModeEnum(), flinkCluster.getProperties(), clusterId, flinkCluster.getId(), - getKubernetesDeployDesc(flinkCluster, "shutdown")); - Future future = executorService.submit(() -> FlinkClient.shutdown(stopRequest)); + getKubernetesDeploySpec(flinkCluster, "shutdown")); + Future future = executorService.submit(() -> FlinkClient.shutdown(stopRequest)); return future.get(60, TimeUnit.SECONDS); } private DeployResponse deployInternal(FlinkCluster flinkCluster) throws InterruptedException, ExecutionException, TimeoutException { DeployRequest deployRequest = new DeployRequest( - flinkEnvService.getById(flinkCluster.getVersionId()).getFlinkVersion(), + FlinkEnvUtils.version(flinkEnvService.getById(flinkCluster.getVersionId())), flinkCluster.getFlinkDeployModeEnum(), flinkCluster.getProperties(), flinkCluster.getClusterId(), flinkCluster.getId(), - getKubernetesDeployDesc(flinkCluster, "start")); + getKubernetesDeploySpec(flinkCluster, "start")); log.info("Deploy cluster request {}", deployRequest); Future future = executorService.submit(() -> FlinkClient.deploy(deployRequest)); return future.get(60, TimeUnit.SECONDS); @@ -450,14 +451,14 @@ private void checkActiveIfNeeded(FlinkCluster flinkCluster) { } @Nullable - private KubernetesDeployParam getKubernetesDeployDesc( - @Nonnull FlinkCluster flinkCluster, String action) { + private KubernetesDeploySpec getKubernetesDeploySpec( + @Nonnull FlinkCluster flinkCluster, String action) { FlinkDeployMode deployModeEnum = flinkCluster.getFlinkDeployModeEnum(); switch (deployModeEnum) { case YARN_SESSION: break; case KUBERNETES_NATIVE_SESSION: - return new KubernetesDeployParam( + return new KubernetesDeploySpec( flinkCluster.getClusterId(), flinkCluster.getK8sNamespace(), flinkCluster.getK8sConf(), diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkEnvServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkEnvServiceImpl.java index 2076fb420d..d85e014478 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkEnvServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkEnvServiceImpl.java @@ -26,6 +26,7 @@ import org.apache.streampark.console.core.service.FlinkClusterService; import org.apache.streampark.console.core.service.FlinkEnvService; import org.apache.streampark.console.core.service.application.FlinkApplicationInfoService; +import org.apache.streampark.console.core.util.FlinkEnvUtils; import org.apache.commons.lang3.StringUtils; @@ -93,8 +94,7 @@ public boolean create(FlinkEnv version) { long count = this.baseMapper.selectCount(null); version.setIsDefault(count == 0); version.setCreateTime(new Date()); - version.doSetVersion(); - version.doSetFlinkConf(); + FlinkEnvUtils.refresh(version, version.getFlinkHome()); return save(version); } @@ -117,9 +117,7 @@ public void update(FlinkEnv version) { flinkEnv.setDescription(version.getDescription()); flinkEnv.setFlinkName(version.getFlinkName()); if (!version.getFlinkHome().equals(flinkEnv.getFlinkHome())) { - flinkEnv.setFlinkHome(version.getFlinkHome()); - flinkEnv.doSetFlinkConf(); - flinkEnv.doSetVersion(); + FlinkEnvUtils.refresh(flinkEnv, version.getFlinkHome()); } updateById(flinkEnv); } @@ -151,7 +149,7 @@ public FlinkEnv getByIdOrDefault(Long id) { @Override public void syncConf(Long id) { FlinkEnv flinkEnv = getById(id); - flinkEnv.doSetFlinkConf(); + FlinkEnvUtils.sync(flinkEnv); updateById(flinkEnv); } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java index 78b1a552b7..7fb55a8488 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java @@ -17,11 +17,11 @@ package org.apache.streampark.console.core.service.impl; +import org.apache.streampark.common.configuration.JvmOptionsParser; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.CompletableFutureUtils; import org.apache.streampark.common.util.ExceptionUtils; -import org.apache.streampark.common.util.FlinkConfigurationUtils; import org.apache.streampark.console.base.domain.RestRequest; import org.apache.streampark.console.base.exception.ApiAlertException; import org.apache.streampark.console.base.exception.InternalException; @@ -43,13 +43,14 @@ import org.apache.streampark.console.core.service.application.ApplicationLogService; import org.apache.streampark.console.core.service.application.FlinkApplicationConfigService; import org.apache.streampark.console.core.service.application.FlinkApplicationManageService; +import org.apache.streampark.console.core.util.FlinkApplicationConfigUtils; +import org.apache.streampark.console.core.util.FlinkEnvUtils; import org.apache.streampark.console.core.util.ServiceHelper; import org.apache.streampark.console.core.watcher.FlinkAppHttpWatcher; import org.apache.streampark.flink.client.FlinkClient; -import org.apache.streampark.flink.client.bean.JobClientTarget; -import org.apache.streampark.flink.client.bean.SavepointResponse; -import org.apache.streampark.flink.client.bean.SavepointTriggerOptions; -import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; +import org.apache.streampark.flink.client.request.JobClientTarget; +import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.response.SavepointResponse; import org.apache.streampark.flink.util.FlinkUtils; import org.apache.commons.collections.CollectionUtils; @@ -155,10 +156,9 @@ public String getSavePointPath(FlinkApplication appParam) throws Exception { return savepointPath; } - // 3) If the savepoint is not obtained above, try to obtain the savepoint path according to the - // deployment type (remote|on yarn) - // 3.1) At the remote mode, request the flink webui interface to get the savepoint path - // 3.2) At the yarn or k8s mode, then read the savepoint in flink-conf.yml in the bound flink + // 3) If no explicit path is configured, resolve it from the deployment target. Remote + // clusters expose the value through the REST API; YARN and Kubernetes deployments use the + // configuration stored for their bound Flink environment. return getSavepointFromDeployLayer(application); } @@ -228,7 +228,7 @@ public void remove(FlinkApplication appParam) { try { appParam .getFsOperator() - .delete(appParam.getWorkspace().APP_SAVEPOINTS().concat("/").concat(appId.toString())); + .delete(appParam.getWorkspace().savepoints.concat("/").concat(appId.toString())); } catch (Exception e) { log.error(e.getMessage(), e); } @@ -333,59 +333,62 @@ private String getClusterId(FlinkApplication application, FlinkCluster cluster) } /** - * Try to get the savepoint config item from the dynamic properties. + * Returns the savepoint directory declared by a dynamic property. * - * @param dynamicProps dynamic properties string. - * @return the value of the savepoint in the dynamic properties. + * @param dynamicProps JVM-style dynamic properties + * @return configured savepoint directory, or {@code null} when absent */ @VisibleForTesting @Nullable public String getSavepointFromDynamicProps(String dynamicProps) { - return FlinkConfigurationUtils.extractDynamicPropertiesAsJava(dynamicProps).get(SAVEPOINT_DIRECTORY.key()); + return JvmOptionsParser.parse(dynamicProps) + .getOptionalString(SAVEPOINT_DIRECTORY.key()) + .orElse(null); } /** - * Try to obtain the savepoint path If it is a streampark|flinksql type task. See if Application - * conf is configured when the task is defined, if checkpoints are configured and enabled, read - * `state.savepoints.dir`. + * Returns the effective savepoint directory for a managed StreamPark or Flink SQL job. * - * @param application the target application. - * @return the value of the savepoint if existed. + *

The directory is used only when checkpointing is enabled in the effective application + * configuration. + * + * @param application target application + * @return configured savepoint directory, or {@code null} when not applicable */ @VisibleForTesting @Nullable public String getSavepointFromConfig(FlinkApplication application) { - if (!application.isStreamParkType() && !application.isFlinkSql()) { + if (!application.isStreamParkType() && !application.isFlinkSqlJob()) { return null; } FlinkApplicationConfig applicationConfig = configService.getEffective(application.getId()); if (applicationConfig == null) { return null; } - Map configMap = applicationConfig.readConfig(); + Map configMap = FlinkApplicationConfigUtils.read(applicationConfig); return FlinkUtils.isCheckpointEnabled(configMap) ? configMap.get(SAVEPOINT_DIRECTORY.key()) : null; } /** - * Try to obtain the savepoint path according to the eployment type (remote|on yarn). At the - * remote mode, request the flink webui interface to get the savepoint path At the yarn or k8s - * mode, then read the savepoint in flink-conf.yml in the bound flink + * Resolves the savepoint directory from the deployment target. + * + *

Remote deployments read the active cluster configuration. YARN and Kubernetes + * deployments read the configuration captured from their bound Flink environment. * - * @param application the target application. - * @return the value of the savepoint if existed. + * @param application target application + * @return configured savepoint directory, or {@code null} when none is defined */ @VisibleForTesting @Nullable public String getSavepointFromDeployLayer(FlinkApplication application) throws JsonProcessingException { - // At the yarn or k8s mode, then read the savepoint in flink-conf.yml in the bound flink if (!FlinkDeployMode.isRemoteMode(application.getDeployMode())) { FlinkEnv flinkEnv = flinkEnvService.getById(application.getVersionId()); - return flinkEnv.convertFlinkYamlAsMap().get(SAVEPOINT_DIRECTORY.key()); + return FlinkEnvUtils.configuration(flinkEnv).get(SAVEPOINT_DIRECTORY.key()); } - // At the remote mode, request the flink webui interface to get the savepoint path + // Remote deployments use the live cluster configuration reported by the Flink REST API. FlinkCluster cluster = flinkClusterService.getById(application.getFlinkClusterId()); AssertUtils.notNull( cluster, @@ -397,12 +400,12 @@ public String getSavepointFromDeployLayer(FlinkApplication application) throws J return config.isEmpty() ? null : config.get(SAVEPOINT_DIRECTORY.key()); } - /** - * Try get the 'state.checkpoints.num-retained' from the dynamic properties. - */ + /** Returns a valid positive checkpoint-retention override from dynamic properties. */ private Optional tryGetChkNumRetainedFromDynamicProps(String dynamicProps) { String rawCfgValue = - FlinkConfigurationUtils.extractDynamicPropertiesAsJava(dynamicProps).get(MAX_RETAINED_CHECKPOINTS.key()); + JvmOptionsParser.parse(dynamicProps) + .getOptionalString(MAX_RETAINED_CHECKPOINTS.key()) + .orElse(null); if (StringUtils.isBlank(rawCfgValue)) { return Optional.empty(); } @@ -411,24 +414,21 @@ private Optional tryGetChkNumRetainedFromDynamicProps(String dynamicPro if (value > 0) { return Optional.of(value); } - log.warn( - "This value of dynamicProperties key: state.checkpoints.num-retained is invalid, must be greater than 0"); + log.warn("Dynamic property {} must be greater than 0", MAX_RETAINED_CHECKPOINTS.key()); } catch (NumberFormatException e) { - log.error( - "This value of dynamicProperties key: state.checkpoints.num-retained invalid, must be number"); + log.warn("Dynamic property {} must be numeric", MAX_RETAINED_CHECKPOINTS.key()); } return Optional.empty(); } - /** - * Try get the 'state.checkpoints.num-retained' from the flink env. - */ + /** Returns the checkpoint-retention limit from the bound Flink environment. */ private int getChkNumRetainedFromFlinkEnv( @Nonnull FlinkEnv flinkEnv, @Nonnull FlinkApplication application) { - String flinkConfNumRetained = flinkEnv.convertFlinkYamlAsMap().get(MAX_RETAINED_CHECKPOINTS.key()); + String flinkConfNumRetained = + FlinkEnvUtils.configuration(flinkEnv).get(MAX_RETAINED_CHECKPOINTS.key()); if (StringUtils.isBlank(flinkConfNumRetained)) { log.info( - "The application: {} is not set {} in dynamicProperties or value is invalid, and flink-conf.yaml is the same problem of flink env: {}, default value: {} will be use.", + "The application: {} does not define a valid {} in dynamic properties or the Flink configuration for environment {}; using default value {}.", application.getJobName(), MAX_RETAINED_CHECKPOINTS.key(), flinkEnv.getFlinkHome(), @@ -441,13 +441,13 @@ private int getChkNumRetainedFromFlinkEnv( return value; } log.warn( - "The value of key: state.checkpoints.num-retained in flink-conf.yaml is invalid, must be greater than 0, default value: {} will be used", + "The value of state.checkpoints.num-retained in the Flink configuration must be greater than 0; using default value {}", MAX_RETAINED_CHECKPOINTS.defaultValue()); } catch (NumberFormatException e) { log.error( - "The value of key: state.checkpoints.num-retained in flink-conf.yaml is invalid, must be number, flink env: {}, default value: {} will be used", + "The value of state.checkpoints.num-retained in Flink environment {} must be numeric; using default value {}", flinkEnv.getFlinkHome(), - flinkConfNumRetained); + MAX_RETAINED_CHECKPOINTS.defaultValue()); } return MAX_RETAINED_CHECKPOINTS.defaultValue(); } @@ -502,10 +502,11 @@ private TriggerSavepointRequest renderTriggerSavepointRequest( return new TriggerSavepointRequest( application.getId(), - flinkEnv.getFlinkVersion(), + FlinkEnvUtils.version(flinkEnv), application.getDeployModeEnum(), properties, new JobClientTarget(clusterId, application.getJobId(), application.getK8sNamespace()), - new SavepointTriggerOptions(customSavepoint, nativeFormat)); + customSavepoint, + nativeFormat); } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java index 84a7b899f7..0790404acd 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java @@ -19,7 +19,6 @@ import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.ExceptionUtils; import org.apache.streampark.console.base.domain.RestRequest; import org.apache.streampark.console.base.mybatis.pager.MybatisPager; import org.apache.streampark.console.core.entity.FlinkApplication; @@ -32,6 +31,7 @@ import org.apache.streampark.console.core.service.FlinkEnvService; import org.apache.streampark.console.core.service.FlinkSqlService; import org.apache.streampark.console.core.service.application.FlinkApplicationBackupService; +import org.apache.streampark.console.core.util.FlinkEnvUtils; import org.apache.streampark.flink.core.FlinkSqlValidationResult; import org.apache.streampark.flink.proxy.FlinkShimsProxy; @@ -46,6 +46,7 @@ import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import java.io.IOException; import java.lang.reflect.Method; import java.util.List; import java.util.Optional; @@ -72,7 +73,7 @@ public class FlinkSqlServiceImpl extends ServiceImpl public FlinkSql getEffective(Long appId, boolean decode) { FlinkSql flinkSql = baseMapper.getEffective(appId); if (flinkSql != null && decode) { - flinkSql.setSql(DeflaterUtils.toPlainText(flinkSql.getSql())); + flinkSql.decode(); } return flinkSql; } @@ -90,7 +91,7 @@ public FlinkSql getLatestFlinkSql(Long appId, boolean decode) { .map( flinkSql -> { if (decode) { - flinkSql.setSql(DeflaterUtils.toPlainText(flinkSql.getSql())); + flinkSql.decode(); } return flinkSql; }) @@ -101,7 +102,7 @@ public FlinkSql getLatestFlinkSql(Long appId, boolean decode) { public void create(FlinkSql flinkSql) { Integer version = this.baseMapper.getLatestVersion(flinkSql.getAppId()); flinkSql.setVersion(version == null ? 1 : version + 1); - flinkSql.setSql(DeflaterUtils.compressForStorage(flinkSql.getSql())); + flinkSql.setSql(DeflaterUtils.zipString(flinkSql.getSql())); this.save(flinkSql); this.setCandidate(CandidateTypeEnum.NEW, flinkSql.getAppId(), flinkSql.getId()); } @@ -182,7 +183,7 @@ public void rollback(FlinkApplication application) { public FlinkSqlValidationResult verifySql(String sql, Long versionId) { FlinkEnv flinkEnv = flinkEnvService.getById(versionId); return FlinkShimsProxy.proxyVerifySql( - flinkEnv.getFlinkVersion(), + FlinkEnvUtils.version(flinkEnv), classLoader -> { try { Class clazz = classLoader.loadClass(FLINKSQL_VALIDATOR_CLASS); @@ -194,12 +195,10 @@ public FlinkSqlValidationResult verifySql(String sql, Long versionId) { } return FlinkShimsProxy.getObject( this.getClass().getClassLoader(), result, FlinkSqlValidationResult.class); - } catch (Throwable e) { - log.error( - "verifySql invocationTargetException: {}", - ExceptionUtils.stringifyException(e)); + } catch (ReflectiveOperationException | IOException e) { + throw new IllegalStateException( + "Failed to validate SQL with Flink " + flinkEnv.getVersion(), e); } - return null; }); } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java index ff89feb79a..f00dd0a726 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java @@ -17,13 +17,13 @@ package org.apache.streampark.console.core.service.impl; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.InternalConfigHolder; -import org.apache.streampark.common.conf.Workspace; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.CompletableFutureUtils; import org.apache.streampark.common.util.FileUtils; +import org.apache.streampark.console.base.config.ConsoleOptions; import org.apache.streampark.console.base.domain.RestRequest; import org.apache.streampark.console.base.exception.ApiAlertException; import org.apache.streampark.console.base.exception.ApiDetailException; @@ -43,7 +43,6 @@ import org.apache.streampark.console.core.watcher.FlinkAppHttpWatcher; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.configuration.MemorySize; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; @@ -374,7 +373,8 @@ public ProjectBuildLogResult getBuildLog(Long id, Long startOffset) { startOffset = 0L; } try { - long maxSize = MemorySize.parse(InternalConfigHolder.get(CommonConfig.READ_LOG_MAX_SIZE())).getBytes(); + long maxSize = + GlobalConfiguration.current().get(ConsoleOptions.BUILD_LOG_READ_MAX_SIZE).bytes(); if (startOffset == null) { fileContent = FileUtils.readEndOfFile(logFile, maxSize); } else { @@ -396,7 +396,7 @@ public ProjectBuildLogResult getBuildLog(Long id, Long startOffset) { } private String getBuildLogPath(Long projectId) { - return String.format("%s/%s/build.log", Workspace.PROJECT_BUILD_LOG_PATH(), projectId); + return String.format("%s/%s/build.log", Workspace.PROJECT_BUILD_LOG_PATH, projectId); } @Override diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java index f770e80edb..ba68a95b77 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java @@ -20,6 +20,7 @@ import org.apache.streampark.common.util.HadoopConfigUtils; import org.apache.streampark.common.util.HadoopUtils; import org.apache.streampark.common.util.YarnUtils; +import org.apache.streampark.console.base.util.WebUtils; import org.apache.streampark.console.core.entity.ApplicationLog; import org.apache.streampark.console.core.entity.FlinkApplication; import org.apache.streampark.console.core.entity.FlinkCluster; @@ -27,41 +28,24 @@ import org.apache.streampark.console.core.service.FlinkClusterService; import org.apache.streampark.console.core.service.ProxyService; import org.apache.streampark.console.core.watcher.FlinkK8sWatcherWrapper; -import org.apache.streampark.flink.kubernetes.FlinkK8sWatcher; +import org.apache.streampark.flink.kubernetes.FlinkKubernetesWatcher; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.security.UserGroupInformation; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.web.client.RestTemplateBuilder; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.http.client.ClientHttpResponse; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.springframework.stereotype.Service; -import org.springframework.web.client.DefaultResponseErrorHandler; -import org.springframework.web.client.RestClientException; -import org.springframework.web.client.RestTemplate; -import javax.annotation.Nonnull; import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; -import java.io.ByteArrayOutputStream; -import java.io.InputStream; -import java.net.URI; +import java.io.IOException; +import java.io.PrintWriter; import java.security.PrivilegedExceptionAction; -import java.util.Enumeration; +/** Routes console proxy requests to Flink, YARN, and remote cluster endpoints. */ @Slf4j @Service public class ProxyServiceImpl implements ProxyService { @@ -70,44 +54,28 @@ public class ProxyServiceImpl implements ProxyService { private FlinkClusterService flinkClusterService; @Autowired - private FlinkK8sWatcher flinkK8sWatcher; + private FlinkKubernetesWatcher flinkK8sWatcher; @Autowired private FlinkK8sWatcherWrapper k8sWatcherWrapper; - private final RestTemplate proxyRestTemplate; - - private String httpAuthUsername = ""; - - public ProxyServiceImpl(RestTemplateBuilder restTemplateBuilder) { - this.proxyRestTemplate = - restTemplateBuilder - .errorHandler( - new DefaultResponseErrorHandler() { - - @Override - public void handleError(@Nonnull ClientHttpResponse response) { - // Ignore errors in the Flink Web UI itself, such as 404 errors. - } - }) - .build(); - } - @Override - public ResponseEntity proxyFlink(HttpServletRequest request, FlinkApplication app) throws Exception { - ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE); - String url = null; + public void proxyFlink( + HttpServletRequest request, + HttpServletResponse response, + FlinkApplication app) throws Exception { + String url; switch (app.getDeployModeEnum()) { case YARN_PER_JOB: case YARN_APPLICATION: case YARN_SESSION: - String yarnURL = YarnUtils.getRMWebAppProxyURL(); - url = yarnURL + "/proxy/" + app.getClusterId(); + url = "/proxy/" + app.getClusterId(); url += getRequestURL(request, "/proxy/flink/" + app.getId()); - return proxyYarnRequest(request, url); + proxyYarnRequest(request, response, url); + return; case REMOTE: FlinkCluster cluster = flinkClusterService.getById(app.getFlinkClusterId()); - url = cluster.getAddress(); + url = cluster == null ? null : cluster.getAddress(); break; case KUBERNETES_NATIVE_APPLICATION: case KUBERNETES_NATIVE_SESSION: @@ -118,26 +86,25 @@ public ResponseEntity proxyFlink(HttpServletRequest request, FlinkApplication "unsupported deployMode ".concat(app.getDeployModeEnum().getName())); } - if (url == null) { - ResponseEntity.BodyBuilder builder = ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE); - builder.body("The flink job manager url is not ready"); - return builder.build(); + if (StringUtils.isBlank(url)) { + unavailableResponse(response, "The flink job manager url is not ready"); + return; } - - url += getRequestURL(request, "/proxy/flink/" + app.getId()); - return proxyRequest(request, url); + proxyRequest(request, response, url + getRequestURL(request, "/proxy/flink/" + app.getId())); } @Override - public ResponseEntity proxySpark(HttpServletRequest request, SparkApplication app) throws Exception { - ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE); + public void proxySpark( + HttpServletRequest request, + HttpServletResponse response, + SparkApplication app) throws Exception { switch (app.getDeployModeEnum()) { case YARN_CLIENT: case YARN_CLUSTER: - String yarnURL = YarnUtils.getRMWebAppProxyURL(); - String url = yarnURL + "/proxy/" + app.getClusterId(); + String url = "/proxy/" + app.getClusterId(); url += getRequestURL(request, "/proxy/spark/" + app.getId()); - return proxyYarnRequest(request, url); + proxyYarnRequest(request, response, url); + return; default: throw new UnsupportedOperationException( "unsupported deployMode ".concat(app.getDeployModeEnum().getName())); @@ -145,40 +112,47 @@ public ResponseEntity proxySpark(HttpServletRequest request, SparkApplication } @Override - public ResponseEntity proxyYarn(HttpServletRequest request, ApplicationLog log) throws Exception { - ResponseEntity.BodyBuilder builder = ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE); + public void proxyYarn( + HttpServletRequest request, + HttpServletResponse response, + ApplicationLog log) throws Exception { String yarnId = log.getClusterId(); if (StringUtils.isBlank(yarnId)) { - return builder.body("The yarn application id is null."); + unavailableResponse(response, "The yarn application id is null."); + return; } - String yarnURL = YarnUtils.getRMWebAppProxyURL(); - String url = yarnURL + "/proxy/" + yarnId + "/"; + String url = "/proxy/" + yarnId + "/"; url += getRequestURL(request, "/proxy/yarn/" + log.getId()); - return proxyYarnRequest(request, url); + proxyYarnRequest(request, response, url); } @Override - public ResponseEntity proxyHistory(HttpServletRequest request, ApplicationLog log) throws Exception { - ResponseEntity.BodyBuilder builder = ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE); - + public void proxyHistory( + HttpServletRequest request, + HttpServletResponse response, + ApplicationLog log) throws Exception { String url = log.getTrackingUrl(); if (StringUtils.isBlank(url)) { - return builder.body("The jobManager url is null."); + unavailableResponse(response, "The jobManager url is null."); + return; } - url += getRequestURL(request, "/proxy/history/" + log.getId()); - return proxyRequest(request, url); + proxyRequest(request, response, url + getRequestURL(request, "/proxy/history/" + log.getId())); } @Override - public ResponseEntity proxyFlinkCluster(HttpServletRequest request, Long clusterId) throws Exception { - ResponseEntity.BodyBuilder builder = ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE); + public void proxyFlinkCluster( + HttpServletRequest request, + HttpServletResponse response, + Long clusterId) throws Exception { FlinkCluster cluster = flinkClusterService.getById(clusterId); if (cluster == null) { - return builder.body("The cluster not found."); + unavailableResponse(response, "The cluster not found."); + return; } String url = cluster.getAddress(); if (StringUtils.isBlank(url)) { - return builder.body("The cluster address is invalid."); + unavailableResponse(response, "The cluster address is invalid."); + return; } url += getRequestURL(request, "/proxy/flink_cluster/" + clusterId); @@ -186,68 +160,64 @@ public ResponseEntity proxyFlinkCluster(HttpServletRequest request, Long clus case YARN_PER_JOB: case YARN_APPLICATION: case YARN_SESSION: - return proxyYarnRequest(request, url); + proxyYarnRequest(request, response, url); + return; case REMOTE: case KUBERNETES_NATIVE_APPLICATION: case KUBERNETES_NATIVE_SESSION: - return proxyRequest(request, url); + proxyRequest(request, response, url); + return; default: throw new UnsupportedOperationException( "unsupported deployMode ".concat(cluster.getFlinkDeployModeEnum().getName())); } } - private HttpEntity getRequestEntity(HttpServletRequest request, String url) throws Exception { - HttpHeaders headers = new HttpHeaders(); - Enumeration headerNames = request.getHeaderNames(); - while (headerNames.hasMoreElements()) { - String headerName = headerNames.nextElement(); - headers.set(headerName, request.getHeader(headerName)); - } - - // Ensure the Host header is set correctly. - URI uri = new URI(url); - headers.set("Host", uri.getHost()); - byte[] body = null; - if (request.getInputStream().available() > 0) { - InputStream inputStream = request.getInputStream(); - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - IOUtils.copy(inputStream, byteArrayOutputStream); - body = byteArrayOutputStream.toByteArray(); + /** Proxies a request and converts transport failures into a gateway error. */ + private void proxyRequest( + HttpServletRequest request, + HttpServletResponse response, + String url) throws Exception { + try { + WebUtils.http(url, request, response); + } catch (Exception e) { + log.error("Proxy request failed for {}", request.getRequestURI(), e); + if (!response.isCommitted()) { + response.sendError(HttpStatus.BAD_GATEWAY.value(), "Unable to reach upstream service"); + } } - return new HttpEntity<>(body, headers); } - private ResponseEntity proxyRequest(HttpServletRequest request, String url) throws Exception { - return proxy(request, url, getRequestEntity(request, url)); - } - - private ResponseEntity proxyYarnRequest(HttpServletRequest request, String url) throws Exception { + /** Executes YARN requests under the configured HTTP authentication context. */ + private void proxyYarnRequest( + HttpServletRequest request, + HttpServletResponse response, + String path) throws Exception { + String url = isAbsoluteUrl(path) ? path : YarnUtils.getRMWebAppProxyURL() + path; if (YarnUtils.hasYarnHttpKerberosAuth()) { + final String kerberosUrl = url; UserGroupInformation ugi = HadoopUtils.getUgi(); - HttpEntity requestEntity = getRequestEntity(request, url); - setRestTemplateCredentials(ugi.getShortUserName()); - return ugi.doAs( - (PrivilegedExceptionAction>) () -> proxy(request, url, requestEntity)); + try { + ugi.doAs( + (PrivilegedExceptionAction) () -> { + proxyRequest(request, response, kerberosUrl); + return null; + }); + } catch (Exception e) { + log.error("Kerberos YARN proxy request failed for {}", request.getRequestURI(), e); + if (!response.isCommitted()) { + response.sendError(HttpStatus.BAD_GATEWAY.value(), "Unable to reach YARN service"); + } + } + return; } + if (YarnUtils.hasYarnHttpSimpleAuth()) { String urlTemplate = StringUtils.isNotBlank(request.getQueryString()) ? "%s&user.name=%s" : "%s?user.name=%s"; - return proxyRequest( - request, String.format(urlTemplate, url, HadoopConfigUtils.hadoopUserName())); - } - return proxyRequest(request, url); - } - - private ResponseEntity proxy( - HttpServletRequest request, String url, HttpEntity requestEntity) { - try { - return proxyRestTemplate.exchange( - url, HttpMethod.valueOf(request.getMethod()), requestEntity, byte[].class); - } catch (RestClientException e) { - log.error("Proxy url: {} failed. ", url, e); - return new ResponseEntity<>(HttpStatus.BAD_GATEWAY); + url = String.format(urlTemplate, url, HadoopConfigUtils.hadoopUserName()); } + proxyRequest(request, response, url); } private String getRequestURL(HttpServletRequest request, String replaceString) { @@ -257,28 +227,15 @@ private String getRequestURL(HttpServletRequest request, String replaceString) { return url.replace(replaceString, ""); } - /** - * Configures the RestTemplate's HttpClient connector. This method is primarily used to configure - * the HttpClient authentication information and SSL certificate validation policies. - * - * @param username The username for HTTP basic authentication. - */ - private void setRestTemplateCredentials(String username) { - // Check if the username is not null and has changed since the last configuration - if (username != null && !this.httpAuthUsername.equals(username)) { - // Create a new credentials provider - BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - // Add the username and password for HTTP basic authentication - credentialsProvider.setCredentials( - AuthScope.ANY, new UsernamePasswordCredentials(username, null)); - // Customize the HttpClient with the credentials provider - CloseableHttpClient httpClient = - HttpClients.custom().setDefaultCredentialsProvider(credentialsProvider).build(); - // Set the HttpClient request factory for the RestTemplate - this.proxyRestTemplate.setRequestFactory( - new HttpComponentsClientHttpRequestFactory(httpClient)); - // Update the last known username - this.httpAuthUsername = username; - } + private boolean isAbsoluteUrl(String url) { + return StringUtils.startsWithAny(url, "http://", "https://"); + } + + private void unavailableResponse(HttpServletResponse response, String body) throws IOException { + response.setStatus(HttpStatus.SERVICE_UNAVAILABLE.value()); + response.setContentType("text/plain;charset=UTF-8"); + PrintWriter writer = response.getWriter(); + writer.write(body); + writer.flush(); } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ResourceServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ResourceServiceImpl.java index e4f1bd42ab..91487b2d7f 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ResourceServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ResourceServiceImpl.java @@ -17,8 +17,8 @@ package org.apache.streampark.console.core.service.impl; -import org.apache.streampark.common.conf.Workspace; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.common.fs.LfsOperator; import org.apache.streampark.common.util.ExceptionUtils; @@ -86,8 +86,6 @@ import java.util.jar.JarFile; import java.util.stream.Collectors; -import static org.apache.streampark.common.enums.StorageType.LFS; - @Slf4j @Service @Transactional(propagation = Propagation.SUPPORTS, readOnly = true, rollbackFor = Exception.class) @@ -225,7 +223,7 @@ public void remove(Long id) { String filePath = String.format( "%s/%d/%s", - Workspace.local().APP_UPLOADS(), + Workspace.LOCAL.uploads, findResource.getTeamId(), findResource.getResourceName()); @@ -300,7 +298,7 @@ public ResourceCheckResult checkResource(Resource resourceParam) throws JsonProc @Override public List listHistoryUploadJars() { - return Arrays.stream(LfsOperator.listDir(Workspace.of(LFS).APP_UPLOADS())) + return Arrays.stream(LfsOperator.listDir(Workspace.LOCAL.uploads)) .filter(File::isFile) .sorted(Comparator.comparingLong(File::lastModified).reversed()) .map(File::getName) @@ -466,7 +464,7 @@ private File getResourceJar(Resource resource) throws Exception { } private void transferTeamResource(Long teamId, String resourcePath) { - String teamUploads = String.format("%s/%d", Workspace.local().APP_UPLOADS(), teamId); + String teamUploads = String.format("%s/%d", Workspace.LOCAL.uploads, teamId); if (!FsOperator.lfs().exists(teamUploads)) { FsOperator.lfs().mkdirs(teamUploads); } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java index 9570394fb1..8173db5965 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/task/ProjectBuildTask.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.core.task; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; import org.apache.streampark.common.util.CommandUtils; import org.apache.streampark.common.util.Utils; import org.apache.streampark.console.base.util.GitUtils; diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationBuildPipelineUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationBuildPipelineUtils.java index 4c4c9b98bf..5918dc1a3d 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationBuildPipelineUtils.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationBuildPipelineUtils.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.core.util; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.common.util.ExceptionUtils; @@ -128,7 +128,7 @@ public static void uploadSqlJobDependencies(Dependency dependencyObject, Resourc if (dependencyObject.getJar().isEmpty()) { return; } - String localUploads = Workspace.local().APP_UPLOADS(); + String localUploads = Workspace.LOCAL.uploads; for (String jar : dependencyObject.getJar()) { File localJar = new File(WebUtils.getAppTempDir(), jar); File uploadJar = new File(localUploads, jar); @@ -292,6 +292,6 @@ private static void mergeDependency( .getJar() .forEach( jar -> jarLibs.add( - String.format("%s/%d/%s", Workspace.local().APP_UPLOADS(), teamId, jar))); + String.format("%s/%d/%s", Workspace.LOCAL.uploads, teamId, jar))); } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ApplicationEntitySupport.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationEntitySupport.java similarity index 86% rename from streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ApplicationEntitySupport.java rename to streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationEntitySupport.java index 2e2b87280d..46604f0a1b 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/ApplicationEntitySupport.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationEntitySupport.java @@ -15,16 +15,21 @@ * limitations under the License. */ -package org.apache.streampark.console.core.entity; +package org.apache.streampark.console.core.util; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.enums.StorageType; import org.apache.streampark.common.fs.FsOperator; -import org.apache.streampark.console.core.util.ApplicationEntityUtils; import org.apache.streampark.flink.packer.maven.DependencyInfo; import com.fasterxml.jackson.annotation.JsonIgnore; +/** + * Read-only adapter that exposes derived application values outside persistence entities. + * + *

The default methods delegate calculations to {@link ApplicationEntityUtils}; implementing + * entities provide only their persisted source fields. + */ public interface ApplicationEntitySupport { Long getProjectId(); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationEntityUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationEntityUtils.java index bc964d5e9a..984d23a576 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationEntityUtils.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/ApplicationEntityUtils.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.core.util; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.enums.StorageType; import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.console.core.bean.Dependency; @@ -26,6 +26,7 @@ import lombok.extern.slf4j.Slf4j; +/** Derives runtime paths and state from persisted application fields. */ @Slf4j public final class ApplicationEntityUtils { @@ -33,19 +34,19 @@ private ApplicationEntityUtils() { } public static String distHome(Long projectId, String module) { - String path = String.format("%s/%s/%s", Workspace.APP_LOCAL_DIST(), projectId.toString(), module); + String path = String.format("%s/%s/%s", Workspace.APP_LOCAL_DIST, projectId.toString(), module); log.info("local distHome:{}", path); return path; } public static String localAppHome(Long id) { - String path = String.format("%s/%s", Workspace.local().APP_WORKSPACE(), id.toString()); + String path = String.format("%s/%s", Workspace.LOCAL.workspace, id.toString()); log.info("local appHome:{}", path); return path; } public static String remoteAppHome(Long id) { - String path = String.format("%s/%s", Workspace.remote().APP_WORKSPACE(), id.toString()); + String path = String.format("%s/%s", Workspace.REMOTE.workspace, id.toString()); log.info("remote appHome:{}", path); return path; } @@ -70,6 +71,6 @@ public static FsOperator fsOperator(StorageType storageType) { } public static Workspace workspace(StorageType storageType) { - return Workspace.of(storageType); + return StorageType.LFS == storageType ? Workspace.LOCAL : Workspace.REMOTE; } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/DataTypeConverterUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/DataTypeConverterUtils.java deleted file mode 100644 index b8b6b677bc..0000000000 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/DataTypeConverterUtils.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.console.core.util; - -import org.apache.streampark.console.core.bean.FlinkDataType; - -import org.apache.flink.table.api.DataTypes; -import org.apache.flink.table.types.DataType; -import org.apache.flink.table.types.logical.CharType; -import org.apache.flink.table.types.logical.DecimalType; -import org.apache.flink.table.types.logical.LogicalType; -import org.apache.flink.table.types.logical.VarCharType; - -public class DataTypeConverterUtils { - - public static DataType convertToDataType(FlinkDataType flinkDataType) { - DataType dataType; - - String type = flinkDataType.getType().toLowerCase(); - boolean isNullable = flinkDataType.isNullable(); - Integer precision = flinkDataType.getPrecision(); - Integer scale = flinkDataType.getScale(); - - switch (type) { - // Integer types - case "tinyint": - case "int1": - dataType = DataTypes.TINYINT(); - break; - case "smallint": - case "int2": - dataType = DataTypes.SMALLINT(); - break; - case "int": - case "integer": - case "int4": - dataType = DataTypes.INT(); - break; - case "bigint": - case "int8": - dataType = DataTypes.BIGINT(); - break; - - // Floating-point types - case "float": - case "real": - dataType = DataTypes.FLOAT(); - break; - case "double": - case "float8": - dataType = DataTypes.DOUBLE(); - break; - - // Decimal and Numeric types - case "decimal": - case "numeric": - if (precision != null && scale != null) { - dataType = DataTypes.DECIMAL(precision, scale); - } else { - dataType = DataTypes.DECIMAL(38, 18); // Default precision and scale - } - break; - - // Character types - case "char": - if (precision != null) { - dataType = DataTypes.CHAR(precision); - } else { - dataType = DataTypes.CHAR(1); // Default size - } - break; - case "varchar": - case "string": - case "text": - dataType = DataTypes.STRING(); - break; - - // Binary data types - case "binary": - case "varbinary": - case "blob": - dataType = DataTypes.BYTES(); - break; - - // Date and time types - case "date": - dataType = DataTypes.DATE(); - break; - case "timestamp": - if (precision != null) { - dataType = DataTypes.TIMESTAMP(precision); - } else { - dataType = DataTypes.TIMESTAMP(3); // Default precision - } - break; - case "time": - dataType = DataTypes.TIME(); - break; - - // Boolean type - case "boolean": - case "bool": - dataType = DataTypes.BOOLEAN(); - break; - - // JSON and other types - case "json": - dataType = DataTypes.STRING(); // JSON as STRING in Flink - break; - case "uuid": - dataType = DataTypes.STRING(); // UUID as STRING - break; - - // Default case for unsupported types - default: - throw new IllegalArgumentException("Unsupported type: " + type); - } - - // Apply nullability - return isNullable ? dataType.nullable() : dataType.notNull(); - } - - public static FlinkDataType convertorToFlinkDataType(DataType dataType) { - LogicalType logicalType = dataType.getLogicalType(); - boolean isNullable = dataType.getLogicalType().isNullable(); - String typeName = logicalType.getTypeRoot().name().toLowerCase(); - Integer precision = null; - Integer scale = null; - - switch (logicalType.getTypeRoot()) { - case CHAR: - case VARCHAR: - if (logicalType instanceof CharType) { - precision = ((CharType) logicalType).getLength(); - } else if (logicalType instanceof VarCharType) { - precision = ((VarCharType) logicalType).getLength(); - } - break; - case DECIMAL: - if (logicalType instanceof DecimalType) { - precision = ((DecimalType) logicalType).getPrecision(); - scale = ((DecimalType) logicalType).getScale(); - } - break; - case TINYINT: - case SMALLINT: - case INTEGER: - case BIGINT: - case FLOAT: - case DOUBLE: - case DATE: - case TIME_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITHOUT_TIME_ZONE: - case TIMESTAMP_WITH_LOCAL_TIME_ZONE: - case BOOLEAN: - case BINARY: - case VARBINARY: - case ARRAY: - case MULTISET: - case MAP: - case ROW: - case RAW: - case NULL: - case SYMBOL: - case UNRESOLVED: - case INTERVAL_YEAR_MONTH: - case INTERVAL_DAY_TIME: - case TIMESTAMP_WITH_TIME_ZONE: - case DISTINCT_TYPE: - case STRUCTURED_TYPE: - // case JSON: - // case UUID: - // These types do not have precision or scale - break; - default: - throw new IllegalArgumentException("Unsupported type: " + logicalType); - } - - return new FlinkDataType(typeName, isNullable, precision, scale); - } -} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkApplicationConfigUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkApplicationConfigUtils.java new file mode 100644 index 0000000000..7314ee3778 --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkApplicationConfigUtils.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.util; + +import org.apache.streampark.common.configuration.ConfigException; +import org.apache.streampark.common.configuration.ConfigPrefix; +import org.apache.streampark.common.configuration.ConfigurationFormat; +import org.apache.streampark.common.configuration.ConfigurationParser; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.option.ApplicationOptions; +import org.apache.streampark.common.util.DeflaterUtils; +import org.apache.streampark.console.core.entity.FlinkApplication; +import org.apache.streampark.console.core.entity.FlinkApplicationConfig; +import org.apache.streampark.console.core.enums.ConfigFileTypeEnum; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** Converts persisted application configuration into runtime and transport representations. */ +public final class FlinkApplicationConfigUtils { + + private static final List USER_PREFIXES = + Arrays.asList( + FlinkOptions.OPTION_PREFIX, + FlinkOptions.PROPERTY_PREFIX, + FlinkOptions.TABLE_PREFIX, + ApplicationOptions.APPLICATION_PREFIX, + ApplicationOptions.SQL_PREFIX); + + private FlinkApplicationConfigUtils() { + } + + /** Copies decoded configuration to a Flink application request. */ + public static void applyTo( + FlinkApplicationConfig config, + FlinkApplication application) { + application.setConfig(encodedContent(config)); + application.setConfigId(config.getId()); + application.setFormat(config.getFormat()); + } + + /** Reads stored user configuration without StreamPark namespace prefixes. */ + public static Map read(FlinkApplicationConfig config) { + ConfigurationFormat format = formatOf(config.getFormat()); + if (format == null) { + return new LinkedHashMap<>(); + } + + Map parsed = + ConfigurationParser.parse( + DeflaterUtils.unzipString(config.getContent()), + format, + "Flink application configuration " + config.getId()) + .toMap(); + Map result = new LinkedHashMap<>(); + parsed.forEach( + (key, value) -> { + String normalizedKey = stripPrefix(key); + if (result.putIfAbsent(normalizedKey, value) != null) { + throw new ConfigException( + "Duplicate application configuration key: " + normalizedKey); + } + }); + return result; + } + + private static String encodedContent(FlinkApplicationConfig config) { + String content = DeflaterUtils.unzipString(config.getContent()); + return Base64.getEncoder().encodeToString(content.getBytes(StandardCharsets.UTF_8)); + } + + private static ConfigurationFormat formatOf(Integer fileType) { + ConfigFileTypeEnum format = ConfigFileTypeEnum.of(fileType); + if (format == null) { + return null; + } + switch (format) { + case YAML: + return ConfigurationFormat.YAML; + case PROPERTIES: + return ConfigurationFormat.PROPERTIES; + case HOCON: + return ConfigurationFormat.HOCON; + default: + return null; + } + } + + private static String stripPrefix(String key) { + for (ConfigPrefix prefix : USER_PREFIXES) { + if (prefix.matches(key)) { + return prefix.stripFrom(key); + } + } + return key; + } +} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkEnvUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkEnvUtils.java new file mode 100644 index 0000000000..ab098ed82d --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkEnvUtils.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.util; + +import org.apache.streampark.common.configuration.ConfigException; +import org.apache.streampark.common.core.FlinkVersion; +import org.apache.streampark.common.util.DeflaterUtils; +import org.apache.streampark.common.util.FlinkConfigurationUtils; +import org.apache.streampark.console.base.exception.ApiAlertException; +import org.apache.streampark.console.base.exception.ApiDetailException; +import org.apache.streampark.console.core.entity.FlinkEnv; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.Objects; +import java.util.Properties; + +/** + * Handles version discovery and configuration conversion for persisted Flink environments. + * + *

{@link FlinkEnv} remains a database entity. Installation inspection, YAML format metadata, + * compression, and native Flink configuration conversion are kept at this boundary instead of in + * the persistence model. + */ +public final class FlinkEnvUtils { + + private static final String LEGACY_PREFIX = "legacy:"; + private static final String STANDARD_PREFIX = "standard:"; + + private FlinkEnvUtils() { + } + + /** + * Refreshes all installation-derived fields after every value has been validated. + * + * @param environment target persistence entity + * @param flinkHome Flink installation directory + */ + public static void refresh(FlinkEnv environment, String flinkHome) { + FlinkEnv target = requireEnvironment(environment); + FlinkVersion detectedVersion = new FlinkVersion(flinkHome); + String version = detectedVersion.version(); + String scalaVersion = detectedVersion.scalaVersion(); + checkScalaVersion(scalaVersion); + String configuration = readConfiguration(flinkHome, version); + + // Publish the snapshot only after discovery, compatibility checks, and I/O all succeed. + target.setFlinkHome(flinkHome); + target.setVersion(version); + target.setScalaVersion(scalaVersion); + target.setFlinkConf(configuration); + } + + /** Reloads configuration without changing the registered installation metadata. */ + public static void sync(FlinkEnv environment) { + FlinkEnv target = requireEnvironment(environment); + target.setFlinkConf(readConfiguration(target.getFlinkHome(), target.getVersion())); + } + + /** Returns a native Flink version descriptor for the registered installation. */ + public static FlinkVersion version(FlinkEnv environment) { + FlinkEnv target = requireEnvironment(environment); + return new FlinkVersion(target.getFlinkHome()); + } + + /** Parses the stored YAML using the format selected when it was synchronized. */ + public static Map configuration(FlinkEnv environment) { + return FlinkConfigurationUtils.loadConfigurationFromString( + yaml(environment), isStandardYaml(environment)); + } + + /** Returns the parsed environment configuration as Java properties. */ + public static Properties properties(FlinkEnv environment) { + Properties properties = new Properties(); + properties.putAll(configuration(environment)); + return properties; + } + + /** Returns the stored configuration as uncompressed YAML text. */ + public static String yaml(FlinkEnv environment) { + String yaml = DeflaterUtils.unzipString(compressedYaml(environment)); + if (yaml == null) { + throw new IllegalStateException("Stored Flink configuration is not valid compressed data"); + } + return yaml; + } + + /** Returns the compressed payload accepted by the Flink runtime initializer. */ + public static String compressedYaml(FlinkEnv environment) { + String stored = requireConfiguration(environment); + if (stored.startsWith(STANDARD_PREFIX)) { + return stored.substring(STANDARD_PREFIX.length()); + } + if (stored.startsWith(LEGACY_PREFIX)) { + return stored.substring(LEGACY_PREFIX.length()); + } + return stored; + } + + /** Returns whether the stored document uses standard nested YAML. */ + public static boolean isStandardYaml(FlinkEnv environment) { + FlinkEnv target = requireEnvironment(environment); + String stored = requireConfiguration(target); + if (stored.startsWith(STANDARD_PREFIX)) { + return true; + } + if (stored.startsWith(LEGACY_PREFIX)) { + return false; + } + // Rows written before the marker was introduced use the selection rule of their registered + // Flink installation. Newly synchronized rows are independent of later filesystem changes. + return FlinkConfigurationUtils.usesStandardYaml( + Path.of(target.getFlinkHome(), "conf").toString(), target.getVersion()); + } + + private static String readConfiguration(String flinkHome, String flinkVersion) { + try { + File file = + FlinkConfigurationUtils.resolveConfigurationFile( + Path.of(flinkHome, "conf").toString(), flinkVersion); + String yaml = Files.readString(file.toPath(), StandardCharsets.UTF_8); + String prefix = + FlinkConfigurationUtils.isStandardYaml(file) ? STANDARD_PREFIX : LEGACY_PREFIX; + return prefix + DeflaterUtils.zipString(yaml); + } catch (ConfigException e) { + throw new ApiAlertException(e.getMessage(), e); + } catch (IOException e) { + throw new ApiDetailException(e); + } + } + + private static void checkScalaVersion(String flinkScalaVersion) { + String streamParkScalaVersion = scala.util.Properties.versionNumberString(); + if (!streamParkScalaVersion.startsWith(flinkScalaVersion)) { + throw new UnsupportedOperationException( + String.format( + "StreamPark uses Scala %s, but the selected Flink installation uses Scala %s", + streamParkScalaVersion, + flinkScalaVersion)); + } + } + + private static FlinkEnv requireEnvironment(FlinkEnv environment) { + return Objects.requireNonNull(environment, "Flink environment must not be null"); + } + + private static String requireConfiguration(FlinkEnv environment) { + String configuration = requireEnvironment(environment).getFlinkConf(); + if (configuration == null) { + throw new IllegalStateException("Flink configuration has not been initialized"); + } + return configuration; + } +} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilder.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilder.java new file mode 100644 index 0000000000..c6ce7602a5 --- /dev/null +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilder.java @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.util; + +import org.apache.streampark.common.configuration.ConfigException; +import org.apache.streampark.common.configuration.Configuration; +import org.apache.streampark.common.configuration.ConfigurationParser; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.FlinkRunOption; + +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; + +import java.io.PrintStream; +import java.net.URLClassLoader; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Builds the Flink shell fragments consumed by StreamPark's launcher scripts. + * + *

Configuration parsing failures and invalid actions are reported to the caller. Values are + * shell-quoted when they contain characters outside a conservative safe set, so configuration + * content cannot introduce additional shell tokens. + * + *

Stored options form the base launcher command. Explicit program arguments are parsed with the + * same Commons CLI schema and replace stored values by long option name, ensuring that one option + * is emitted at most once. + */ +public final class FlinkShellCommandBuilder { + + private static final Pattern SAFE_SHELL_TOKEN = + Pattern.compile("[A-Za-z0-9_./:@%+=,-]+"); + private static final String PROPERTY_PREFIX = FlinkOptions.PROPERTY_PREFIX.value(); + private static final String OPTION_PREFIX = FlinkOptions.OPTION_PREFIX.value(); + private static final String OPTION_MAIN = + PROPERTY_PREFIX + FlinkOptions.APPLICATION_MAIN_CLASS.key(); + + private static final Options FLINK_OPTIONS = FlinkRunOption.allOptions(); + private static final DefaultParser PARSER = new DefaultParser(); + + private FlinkShellCommandBuilder() { + } + + /** + * Writes a generated shell fragment without adding a line separator. + * + * @param output generated shell fragment + * @param out destination stream + */ + public static void emit(String output, PrintStream out) { + out.print(output); + } + + /** + * Executes one launcher query such as {@code --property }. + * + * @param args launcher action, configuration path, and optional program arguments + * @return generated shell fragment or requested scalar value + * @throws ConfigException when the action, configuration, or launcher arguments are invalid + */ + public static String read(String[] args) { + if (args == null || args.length == 0) { + throw new ConfigException("A launcher action is required"); + } + if ("--vmopt".equals(args[0])) { + return readVmOpt(); + } + if (args.length < 2) { + throw new ConfigException("A configuration file is required for action " + args[0]); + } + return readConfigAction(args[0], args[1], Arrays.copyOfRange(args, 2, args.length)); + } + + private static String readVmOpt() { + ClassLoader classLoader = ClassLoader.getSystemClassLoader(); + if (classLoader instanceof URLClassLoader) { + return ""; + } + return "--add-opens java.base/jdk.internal.loader=ALL-UNNAMED " + + "--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED"; + } + + private static String readConfigAction(String action, String configFile, String[] programArgs) { + Configuration configuration = ConfigurationParser.parse(Path.of(configFile)); + Map values = configuration.toMap(); + switch (action) { + case "--option": + return buildOptionString(values, programArgs); + case "--property": + return buildPropertyString(values); + case "--name": + return readAppName(values); + case "--detached": + return readDetachedMode(values, programArgs); + default: + throw new ConfigException("Unsupported launcher action: " + action); + } + } + + private static String buildOptionString(Map values, String[] programArgs) { + StringBuilder buffer = new StringBuilder(); + try { + org.apache.commons.cli.CommandLine line = + PARSER.parse(FLINK_OPTIONS, getOption(values, programArgs), false); + for (org.apache.commons.cli.Option option : line.getOptions()) { + buffer.append(" -").append(option.getOpt()); + if (option.hasArg()) { + buffer.append(" ").append(shellToken(option.getValue())); + } + } + } catch (org.apache.commons.cli.ParseException e) { + throw new ConfigException("Invalid Flink launcher option", e); + } + String mainClass = values.get(OPTION_MAIN); + if (mainClass != null && !mainClass.isEmpty()) { + buffer.append(" -c ").append(shellToken(mainClass)); + } + return buffer.toString().trim(); + } + + private static String buildPropertyString(Map values) { + StringBuilder buffer = new StringBuilder(); + values.forEach((key, value) -> appendPropertyEntry(buffer, key, value)); + return buffer.toString().trim(); + } + + private static void appendPropertyEntry( + StringBuilder buffer, + String key, + String value) { + if (OPTION_MAIN.equals(key) + || !FlinkOptions.PROPERTY_PREFIX.matches(key) + || value == null + || value.isEmpty()) { + return; + } + String propertyKey = FlinkOptions.PROPERTY_PREFIX.stripFrom(key).trim(); + String propertyValue = value.trim(); + if (FlinkOptions.PIPELINE_NAME.key().equals(propertyKey)) { + propertyValue = propertyValue.replace(" ", "_"); + } + buffer + .append(" -D") + .append(propertyKey) + .append("=") + .append(shellToken(propertyValue)); + } + + private static String readAppName(Map values) { + return values.getOrDefault(PROPERTY_PREFIX + FlinkOptions.PIPELINE_NAME.key(), "").trim(); + } + + private static String readDetachedMode(Map values, String[] programArgs) { + try { + org.apache.commons.cli.CommandLine line = + PARSER.parse(FLINK_OPTIONS, getOption(values, programArgs), false); + boolean detached = + line.hasOption(FlinkRunOption.DETACHED_OPTION.getOpt()) + || line.hasOption(FlinkRunOption.DETACHED_OPTION.getLongOpt()); + return detached ? "Detached" : "Attach"; + } catch (org.apache.commons.cli.ParseException e) { + throw new ConfigException("Invalid Flink detached-mode option", e); + } + } + + static String[] getOption(Map values, String[] args) { + Map optionMap = collectConfiguredOptions(values); + // Program arguments represent the caller's most specific intent and therefore replace + // values collected from the persisted configuration. + mergeProgramArgs(optionMap, args); + return toOptionArray(optionMap); + } + + private static Map collectConfiguredOptions(Map values) { + Map options = new LinkedHashMap<>(); + values.forEach((key, value) -> putConfiguredOption(options, key, value)); + return options; + } + + private static void putConfiguredOption( + Map optionMap, + String key, + String value) { + if (!FlinkOptions.OPTION_PREFIX.matches(key) || value == null || value.isEmpty()) { + return; + } + String optionKey = FlinkOptions.OPTION_PREFIX.stripFrom(key); + if (!FLINK_OPTIONS.hasOption(optionKey)) { + throw new ConfigException("Unknown Flink launcher option in configuration: " + optionKey); + } + if ("true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value)) { + if (Boolean.parseBoolean(value)) { + optionMap.put("-" + optionKey.trim(), true); + } + return; + } + optionMap.put("-" + optionKey.trim(), value); + } + + private static void mergeProgramArgs(Map optionMap, String[] args) { + if (args.length == 0) { + return; + } + try { + org.apache.commons.cli.CommandLine line = PARSER.parse(FLINK_OPTIONS, args, false); + for (org.apache.commons.cli.Option option : line.getOptions()) { + String key = "-" + option.getLongOpt().trim(); + optionMap.put(key, option.hasArg() ? option.getValue() : true); + } + } catch (org.apache.commons.cli.ParseException e) { + throw new ConfigException("Invalid program launcher arguments", e); + } + } + + private static String[] toOptionArray(Map optionMap) { + List result = new ArrayList<>(); + optionMap.forEach( + (key, value) -> { + result.add(key); + if (value instanceof String) { + result.add(value.toString()); + } + }); + return result.toArray(new String[0]); + } + + private static String shellToken(String value) { + if (SAFE_SHELL_TOKEN.matcher(value).matches()) { + return value; + } + // POSIX shells cannot escape a single quote inside single quotes. Close the quoted region, + // emit an escaped quote, and reopen it without creating a second argument. + return "'" + value.replace("'", "'\\''") + "'"; + } +} diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/YarnQueueLabelExpression.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/YarnQueueLabelExpression.java index 101fa20089..a05023081b 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/YarnQueueLabelExpression.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/YarnQueueLabelExpression.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.core.util; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.option.DeploymentOptions; import org.apache.streampark.console.base.exception.ApiAlertException; import org.apache.commons.lang3.StringUtils; @@ -109,8 +109,9 @@ public static Map getQueueLabelMap(String queueLabelExp) { Map queueLabelMap = new HashMap<>(2); yarnQueueLabelExpression .getLabelExpression() - .ifPresent(labelExp -> queueLabelMap.put(ConfigKeys.KEY_YARN_APP_NODE_LABEL(), labelExp)); - queueLabelMap.put(ConfigKeys.KEY_YARN_APP_QUEUE(), yarnQueueLabelExpression.queue); + .ifPresent( + labelExp -> queueLabelMap.put(DeploymentOptions.YARN_NODE_LABEL.key(), labelExp)); + queueLabelMap.put(DeploymentOptions.YARN_QUEUE.key(), yarnQueueLabelExpression.queue); return queueLabelMap; } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkClusterWatcher.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkClusterWatcher.java index 8abab9c0f3..4de04a7bef 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkClusterWatcher.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkClusterWatcher.java @@ -17,13 +17,13 @@ package org.apache.streampark.console.core.watcher; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.InternalConfigHolder; +import org.apache.streampark.common.configuration.GlobalConfiguration; import org.apache.streampark.common.enums.ClusterState; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.util.HadoopUtils; import org.apache.streampark.common.util.HttpClientUtils; import org.apache.streampark.common.util.YarnUtils; +import org.apache.streampark.console.base.config.ConsoleOptions; import org.apache.streampark.console.base.util.JacksonUtils; import org.apache.streampark.console.core.entity.FlinkCluster; import org.apache.streampark.console.core.metrics.flink.Overview; @@ -134,7 +134,8 @@ private void alert(FlinkCluster cluster, ClusterState state) { cluster.setAllJobs(applicationInfoService.countByClusterId(cluster.getId())); cluster.setAffectedJobs( applicationInfoService.countAffectedByClusterId( - cluster.getId(), InternalConfigHolder.get(CommonConfig.SPRING_PROFILES_ACTIVE()))); + cluster.getId(), + GlobalConfiguration.current().get(ConsoleOptions.DATABASE_DIALECT))); cluster.setClusterState(state.getState()); cluster.setEndTime(new Date()); alertService.alert( diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkK8sChangeEventListener.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkK8sChangeEventListener.java index 6e23bf6fc0..6ff5b11808 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkK8sChangeEventListener.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkK8sChangeEventListener.java @@ -27,7 +27,7 @@ import org.apache.streampark.console.core.service.application.FlinkApplicationManageService; import org.apache.streampark.console.core.util.AlertTemplateUtils; import org.apache.streampark.flink.kubernetes.enums.FlinkJobState; -import org.apache.streampark.flink.kubernetes.enums.FlinkK8sDeployMode; +import org.apache.streampark.flink.kubernetes.enums.FlinkKubernetesDeployMode; import org.apache.streampark.flink.kubernetes.event.FlinkClusterMetricChangeEvent; import org.apache.streampark.flink.kubernetes.event.FlinkJobCheckpointChangeEvent; import org.apache.streampark.flink.kubernetes.event.FlinkJobStatusChangeEvent; @@ -116,7 +116,7 @@ public void subscribeJobStatusChange(FlinkJobStatusChangeEvent event) { @Subscribe public void subscribeMetricsChange(FlinkClusterMetricChangeEvent event) { TrackId trackId = event.trackId(); - FlinkDeployMode mode = FlinkK8sDeployMode.toFlinkDeployMode(trackId.executeMode()); + FlinkDeployMode mode = FlinkKubernetesDeployMode.toFlinkDeployMode(trackId.executeMode()); // discard session mode change if (FlinkDeployMode.KUBERNETES_NATIVE_SESSION == mode) { return; diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkK8sWatcherWrapper.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkK8sWatcherWrapper.java index 4e4f09128c..60d53f07af 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkK8sWatcherWrapper.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/watcher/FlinkK8sWatcherWrapper.java @@ -17,16 +17,17 @@ package org.apache.streampark.console.core.watcher; +import org.apache.streampark.common.configuration.JvmOptionsParser; import org.apache.streampark.common.enums.FlinkDeployMode; -import org.apache.streampark.common.util.FlinkConfigurationUtils; import org.apache.streampark.console.core.entity.FlinkApplication; import org.apache.streampark.console.core.entity.FlinkCluster; import org.apache.streampark.console.core.entity.FlinkEnv; import org.apache.streampark.console.core.service.FlinkClusterService; import org.apache.streampark.console.core.service.FlinkEnvService; import org.apache.streampark.console.core.service.application.FlinkApplicationManageService; -import org.apache.streampark.flink.kubernetes.FlinkK8sWatcher; -import org.apache.streampark.flink.kubernetes.FlinkK8sWatcherFactory; +import org.apache.streampark.console.core.util.FlinkEnvUtils; +import org.apache.streampark.flink.kubernetes.FlinkKubernetesWatcher; +import org.apache.streampark.flink.kubernetes.FlinkKubernetesWatcherFactory; import org.apache.streampark.flink.kubernetes.FlinkTrackConfig; import org.apache.streampark.flink.kubernetes.enums.FlinkJobState; import org.apache.streampark.flink.kubernetes.model.TrackId; @@ -81,9 +82,10 @@ public class FlinkK8sWatcherWrapper { /** Register FlinkTrackMonitor bean for tracking flink job on kubernetes. */ @Bean(destroyMethod = "close") - public FlinkK8sWatcher registerFlinkK8sWatcher() { + public FlinkKubernetesWatcher registerFlinkK8sWatcher() { // lazy start tracking monitor - FlinkK8sWatcher flinkK8sWatcher = FlinkK8sWatcherFactory.createInstance(FlinkTrackConfig.fromConfigHub(), true); + FlinkKubernetesWatcher flinkK8sWatcher = + FlinkKubernetesWatcherFactory.createInstance(FlinkTrackConfig.fromConfiguration(), true); initFlinkK8sWatcher(flinkK8sWatcher); /* @@ -97,7 +99,7 @@ public FlinkK8sWatcher registerFlinkK8sWatcher() { return flinkK8sWatcher; } - private void initFlinkK8sWatcher(@Nonnull FlinkK8sWatcher trackMonitor) { + private void initFlinkK8sWatcher(@Nonnull FlinkKubernetesWatcher trackMonitor) { // register change event listener trackMonitor.registerListener(flinkK8sChangeEventListener); // recovery tracking list @@ -126,10 +128,10 @@ private List getK8sWatchingApps() { public TrackId toTrackId(FlinkApplication app) { FlinkEnv flinkEnv = flinkEnvService.getById(app.getVersionId()); - Properties properties = flinkEnv.getFlinkConfig(); + Properties properties = FlinkEnvUtils.properties(flinkEnv); - Map dynamicProperties = FlinkConfigurationUtils - .extractDynamicPropertiesAsJava(app.getDynamicProperties()); + Map dynamicProperties = + JvmOptionsParser.parse(app.getDynamicProperties()).toMap(); String archiveDir = dynamicProperties.get(JobManagerOptions.ARCHIVE_DIR.key()); if (archiveDir != null) { properties.put(JobManagerOptions.ARCHIVE_DIR.key(), archiveDir); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/entity/User.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/entity/User.java index df5d04ef7c..6681c65aa1 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/entity/User.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/entity/User.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.system.entity; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; import org.apache.streampark.console.base.mybatis.entity.BaseEntity; import org.apache.streampark.console.core.enums.LoginTypeEnum; import org.apache.streampark.console.core.enums.UserTypeEnum; diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/runner/StartedUpRunner.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/runner/StartedUpRunner.java index 3f63fc6d42..ca8d4c9a41 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/runner/StartedUpRunner.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/system/runner/StartedUpRunner.java @@ -48,7 +48,7 @@ public void run(ApplicationArguments args) { System.out.println(" ___/ / /_/ / / __/ /_/ / / / / / / /_/ / /_/ / / / ,< "); System.out.println(" /____/\\__/_/ \\___/\\__,_/_/ /_/ /_/ ____/\\__,_/_/ /_/|_| "); System.out.println(" /_/ \n\n"); - System.out.println(" Version: 3.0.0-SNAPSHOT "); + System.out.println(" Version: 3.0.0 "); System.out.println(" WebSite: https://streampark.apache.org "); System.out.println(" GitHub : https://github.com/apache/streampark "); System.out.println(" Info : streampark-console start successful "); diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/SpringIntegrationTestBase.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/SpringIntegrationTestBase.java index 45b8691935..3b24dac71c 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/SpringIntegrationTestBase.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/SpringIntegrationTestBase.java @@ -17,8 +17,8 @@ package org.apache.streampark.console; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.option.CoreOptions; +import org.apache.streampark.common.configuration.option.WorkspaceOptions; import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.SystemPropertyUtils; @@ -108,15 +108,15 @@ public static void init(@TempDir File tempPath) throws IOException { Path localWorkspace = Files.createDirectories(new File(tempAbsPath, DEFAULT_LOCAL_WORKSPACE_DIR_NAME).toPath()); appHome = new File(tempAbsPath, DEFAULT_APP_HOME_DIR_NAME).getAbsolutePath(); - System.setProperty(ConfigKeys.KEY_APP_HOME(), appHome); + System.setProperty(CoreOptions.APP_HOME.key(), appHome); System.setProperty( - CommonConfig.STREAMPARK_WORKSPACE_LOCAL().key(), + WorkspaceOptions.LOCAL_ROOT.key(), localWorkspace.toAbsolutePath().toString()); LOG.info( "Complete mock EnvInitializer init, app home: {}, {}: {}", appHome, - CommonConfig.STREAMPARK_WORKSPACE_LOCAL().key(), + WorkspaceOptions.LOCAL_ROOT.key(), localWorkspace.toAbsolutePath()); } diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/SpringUnitTestBase.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/SpringUnitTestBase.java index 625afd6b61..d3449a17e8 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/SpringUnitTestBase.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/SpringUnitTestBase.java @@ -17,8 +17,8 @@ package org.apache.streampark.console; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.option.CoreOptions; +import org.apache.streampark.common.configuration.option.WorkspaceOptions; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.console.core.entity.FlinkApplication; import org.apache.streampark.console.core.entity.FlinkCluster; @@ -43,13 +43,12 @@ import java.nio.file.Files; import java.nio.file.Path; -/** base tester. */ +/** Shared Spring context and isolated workspace for Console service tests. */ @Slf4j @EnableScheduling @ActiveProfiles("test") @AutoConfigureTestEntityManager -@SpringBootTest(classes = StreamParkConsoleBootstrap.class, webEnvironment = SpringBootTest.WebEnvironment.DEFINED_PORT, properties = { - "server.port=10000", +@SpringBootTest(classes = StreamParkConsoleBootstrap.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = { "spring.application.name=Apache StreamPark", "spring.main.banner-mode=false", "spring.aop.proxy-target-class=true", @@ -84,9 +83,9 @@ public static void init(@TempDir File tempPath) throws IOException { String mockedHome = tempPath.getAbsolutePath(); Path localWorkspace = Files.createDirectories(new File(mockedHome + "/localWorkspace").toPath()); - System.setProperty(ConfigKeys.KEY_APP_HOME(), mockedHome); + System.setProperty(CoreOptions.APP_HOME.key(), mockedHome); System.setProperty( - CommonConfig.STREAMPARK_WORKSPACE_LOCAL().key(), + WorkspaceOptions.LOCAL_ROOT.key(), localWorkspace.toAbsolutePath().toString()); Files.createDirectories(new File(mockedHome + "/temp").toPath()); @@ -94,7 +93,7 @@ public static void init(@TempDir File tempPath) throws IOException { LOG.info( "Complete mock EnvInitializer init, app home: {}, {}: {}", tempPath.getAbsolutePath(), - CommonConfig.STREAMPARK_WORKSPACE_LOCAL().key(), + WorkspaceOptions.LOCAL_ROOT.key(), localWorkspace.toAbsolutePath()); } diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/config/SpringConfigurationInitializerTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/config/SpringConfigurationInitializerTest.java new file mode 100644 index 0000000000..888f531e7b --- /dev/null +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/config/SpringConfigurationInitializerTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.base.config; + +import org.apache.streampark.common.configuration.ConfigException; +import org.apache.streampark.common.configuration.ConfigOrigin; +import org.apache.streampark.common.configuration.ConfigSource; +import org.apache.streampark.common.configuration.Configuration; +import org.apache.streampark.common.configuration.DataSize; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.option.DockerOptions; +import org.apache.streampark.common.configuration.option.HadoopOptions; +import org.apache.streampark.common.configuration.option.YarnOptions; +import org.apache.streampark.flink.kubernetes.configuration.KubernetesOptions; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.mock.env.MockEnvironment; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the explicit integration boundary between Spring and typed server configuration. */ +class SpringConfigurationInitializerTest { + + private Configuration previousConfiguration; + private String previousHadoopUser; + + @BeforeEach + void captureGlobalConfiguration() { + previousConfiguration = GlobalConfiguration.current(); + previousHadoopUser = System.getProperty(HadoopOptions.HADOOP_USER_NAME_PROPERTY); + } + + @AfterEach + void restoreGlobalConfiguration() { + GlobalConfiguration.update(previousConfiguration); + if (previousHadoopUser == null) { + System.clearProperty(HadoopOptions.HADOOP_USER_NAME_PROPERTY); + } else { + System.setProperty(HadoopOptions.HADOOP_USER_NAME_PROPERTY, previousHadoopUser); + } + } + + @Test + void publishDeclaredSpringOverrides() { + MockEnvironment environment = new MockEnvironment() + .withProperty(HadoopOptions.HADOOP_USER_NAME_PROPERTY, "spring-hadoop-user") + .withProperty(HadoopOptions.KERBEROS_ENABLED.key(), "true") + .withProperty(YarnOptions.PROXY_URL.key(), "https://rm.example.test") + .withProperty(DockerOptions.MAX_CONNECTIONS.key(), "37") + .withProperty(ConsoleOptions.BUILD_LOG_READ_MAX_SIZE.key(), "2 MiB") + .withProperty(KubernetesOptions.INGRESS_CLASS.key(), "internal-nginx") + .withProperty("spring.unrelated.property", "must-not-leak"); + + Configuration configuration = new SpringConfigurationInitializer().initialize(environment); + + assertThat(configuration.get(HadoopOptions.KERBEROS_ENABLED)).isTrue(); + assertThat(System.getProperty(HadoopOptions.HADOOP_USER_NAME_PROPERTY)) + .isEqualTo("spring-hadoop-user"); + assertThat(configuration.get(YarnOptions.PROXY_URL)).isEqualTo("https://rm.example.test"); + assertThat(configuration.get(DockerOptions.MAX_CONNECTIONS)).isEqualTo(37); + assertThat(configuration.get(ConsoleOptions.BUILD_LOG_READ_MAX_SIZE)) + .isEqualTo(DataSize.ofMebiBytes(2)); + assertThat(configuration.get(KubernetesOptions.INGRESS_CLASS)).isEqualTo("internal-nginx"); + assertThat(configuration.containsKey("spring.unrelated.property")).isFalse(); + assertThat(configuration.origin(DockerOptions.MAX_CONNECTIONS.key())) + .contains(ConfigOrigin.of(ConfigSource.RUNTIME, "resolved Spring environment")); + } + + @Test + void rejectInvalidSpringOverrides() { + MockEnvironment environment = new MockEnvironment() + .withProperty(DockerOptions.MAX_CONNECTIONS.key(), "0"); + + assertThatThrownBy(() -> new SpringConfigurationInitializer().initialize(environment)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining(DockerOptions.MAX_CONNECTIONS.key()); + assertThat(GlobalConfiguration.current()).isSameAs(previousConfiguration); + } +} diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/DependencyUtilsTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/DependencyUtilsTest.java index 44246e7028..434c313bef 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/DependencyUtilsTest.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/DependencyUtilsTest.java @@ -17,8 +17,8 @@ package org.apache.streampark.console.base.util; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.InternalConfigHolder; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.option.WorkspaceOptions; import org.apache.streampark.console.core.bean.FlinkConnector; import org.apache.streampark.flink.packer.maven.Artifact; import org.apache.streampark.flink.packer.maven.MavenTool; @@ -56,7 +56,7 @@ public void resolveFlinkConnector() throws Exception { Artifact artifact = new Artifact("com.ververica", "flink-connector-mysql-cdc", "2.4.1", null); - InternalConfigHolder.set(CommonConfig.STREAMPARK_WORKSPACE_LOCAL(), "~/tmp"); + GlobalConfiguration.set(WorkspaceOptions.LOCAL_ROOT, "~/tmp", "test"); List files = MavenTool.resolveArtifacts(artifact); if (files.isEmpty()) { diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/ShaHashUtilsTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/ShaHashUtilsTest.java index f85601359f..7b5806ddf3 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/ShaHashUtilsTest.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/ShaHashUtilsTest.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.base.util; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/WebUtilsTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/WebUtilsTest.java new file mode 100644 index 0000000000..f66622a37b --- /dev/null +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/WebUtilsTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.base.util; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockHttpServletRequest; +import org.springframework.mock.web.MockHttpServletResponse; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +class WebUtilsTest { + + private HttpServer server; + private AtomicReference method; + private AtomicReference requestBody; + private AtomicReference forwardedHeader; + private AtomicReference skippedHeader; + private String endpoint; + + @BeforeEach + void setUp() throws IOException { + method = new AtomicReference<>(); + requestBody = new AtomicReference<>(); + forwardedHeader = new AtomicReference<>(); + skippedHeader = new AtomicReference<>(); + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/proxy", + exchange -> { + captureRequest(exchange); + byte[] responseBody = "upstream response".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("X-Upstream", "ok"); + exchange.getResponseHeaders().add("Allow", "GET, POST"); + exchange.getResponseHeaders().add("Connection", "X-Upstream-Hop"); + exchange.getResponseHeaders().add("X-Upstream-Hop", "internal"); + exchange.sendResponseHeaders(201, responseBody.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(responseBody); + } + }); + server.start(); + endpoint = "http://127.0.0.1:" + server.getAddress().getPort() + "/proxy"; + } + + @AfterEach + void tearDown() { + server.stop(0); + } + + @Test + void forwardProxyRequestAndResponse() throws Exception { + MockHttpServletRequest request = new MockHttpServletRequest("POST", "/proxy"); + request.setContentType("application/json"); + request.setContent("{\"name\":\"streampark\"}".getBytes(StandardCharsets.UTF_8)); + request.addHeader("X-Request-Id", "request-1"); + request.addHeader("X-Request-Hop", "internal"); + request.addHeader("Origin", "http://console.example"); + request.addHeader("Connection", "keep-alive, X-Request-Hop"); + MockHttpServletResponse response = new MockHttpServletResponse(); + + WebUtils.http(endpoint, request, response); + + assertThat(method.get()).isEqualTo("POST"); + assertThat(requestBody.get()).isEqualTo("{\"name\":\"streampark\"}"); + assertThat(forwardedHeader.get()).isEqualTo("request-1"); + assertThat(skippedHeader.get()).isNull(); + assertThat(response.getStatus()).isEqualTo(201); + assertThat(response.getHeader("X-Upstream")).isEqualTo("ok"); + assertThat(response.getHeader("Allow")).isEqualTo("GET, POST"); + assertThat(response.getHeader("X-Upstream-Hop")).isNull(); + assertThat(response.getContentAsString()).isEqualTo("upstream response"); + } + + private void captureRequest(HttpExchange exchange) throws IOException { + method.set(exchange.getRequestMethod()); + requestBody.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8)); + forwardedHeader.set(exchange.getRequestHeaders().getFirst("X-Request-Id")); + skippedHeader.set(exchange.getRequestHeaders().getFirst("X-Request-Hop")); + } +} diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/assembler/FlinkApplicationAssemblerTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/assembler/FlinkApplicationAssemblerTest.java index 255eba8c5f..5052ca271b 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/assembler/FlinkApplicationAssemblerTest.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/assembler/FlinkApplicationAssemblerTest.java @@ -84,6 +84,7 @@ void shouldConvertEntityToResponse() { app.setId(10L); app.setJobName("sql-job"); app.setState(0); + app.setFlinkSql("encoded-sql"); FlinkAppResponse response = FlinkApplicationAssembler.toResponse(app); @@ -91,6 +92,7 @@ void shouldConvertEntityToResponse() { Assertions.assertEquals(10L, response.getId()); Assertions.assertEquals("sql-job", response.getJobName()); Assertions.assertEquals(0, response.getState()); + Assertions.assertEquals("encoded-sql", response.getFlinkSql()); } @Test diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/entity/FlinkSqlTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/entity/FlinkSqlTest.java new file mode 100644 index 0000000000..f3234ceb2c --- /dev/null +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/entity/FlinkSqlTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.entity; + +import org.apache.streampark.common.util.DeflaterUtils; + +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class FlinkSqlTest { + + private static final String SQL = "SELECT * FROM source;"; + + @Test + void shouldApplyPersistedSqlToApplication() { + FlinkSql flinkSql = flinkSql(DeflaterUtils.zipString(SQL)); + FlinkApplication application = new FlinkApplication(); + + flinkSql.decode(); + flinkSql.applyToApplication(application); + + assertThat(decodeTransportValue(application.getFlinkSql())).isEqualTo(SQL); + assertThat(application.getSqlId()).isEqualTo(flinkSql.getId()); + } + + @Test + void shouldApplyPlainSqlToApplication() { + FlinkApplication application = new FlinkApplication(); + + flinkSql(SQL).applyToApplication(application); + + assertThat(decodeTransportValue(application.getFlinkSql())).isEqualTo(SQL); + } + + @Test + void shouldEncodePersistedSqlForHistoryApi() { + FlinkSql flinkSql = flinkSql(DeflaterUtils.zipString(SQL)); + + flinkSql.base64Encode(); + + assertThat(decodeTransportValue(flinkSql.getSql())).isEqualTo(SQL); + } + + @Test + void shouldRejectCorruptPersistedSql() { + FlinkSql flinkSql = flinkSql("not-compressed-sql"); + + assertThatThrownBy(flinkSql::decode) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Failed to decompress Flink SQL id=100002"); + } + + private static FlinkSql flinkSql(String sql) { + FlinkSql flinkSql = new FlinkSql(); + flinkSql.setId(100002L); + flinkSql.setSql(sql); + return flinkSql; + } + + private static String decodeTransportValue(String value) { + return new String(Base64.getDecoder().decode(value), StandardCharsets.UTF_8); + } +} diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/FlinkSavepointServiceTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/FlinkSavepointServiceTest.java index 54eece310c..b81378ba11 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/FlinkSavepointServiceTest.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/FlinkSavepointServiceTest.java @@ -21,6 +21,7 @@ import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; import org.apache.streampark.common.util.DeflaterUtils; +import org.apache.streampark.common.util.FlinkConfigurationUtils; import org.apache.streampark.console.SpringUnitTestBase; import org.apache.streampark.console.core.entity.FlinkApplication; import org.apache.streampark.console.core.entity.FlinkApplicationConfig; @@ -31,23 +32,23 @@ import org.apache.streampark.console.core.service.application.FlinkApplicationConfigService; import org.apache.streampark.console.core.service.application.FlinkApplicationManageService; import org.apache.streampark.console.core.service.impl.FlinkSavepointServiceImpl; +import org.apache.streampark.console.core.util.FlinkEnvUtils; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.fasterxml.jackson.core.JsonProcessingException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.springframework.beans.factory.annotation.Autowired; +import java.nio.file.Files; +import java.nio.file.Path; + import static org.apache.flink.configuration.CheckpointingOptions.SAVEPOINT_DIRECTORY; import static org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions.CHECKPOINTING_INTERVAL; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** - * Test class for the implementation {@link - * FlinkSavepointServiceImpl} of {@link - * SavepointService}. - */ +/** Tests savepoint configuration resolution performed by {@link FlinkSavepointServiceImpl}. */ class FlinkSavepointServiceTest extends SpringUnitTestBase { @Autowired @@ -76,12 +77,8 @@ void cleanTestRecordsInDatabase() { applicationManageService.remove(new QueryWrapper<>()); } - /** - * This part will be migrated into the corresponding test cases about - * PropertiesUtils.extractDynamicPropertiesAsJava. - */ @Test - void testGetSavepointFromDynamicProps() { + void readSavepointFromDynamicProps() { String propsWithEmptyTargetValue = "-Dexecution.checkpointing.savepoint-dir="; String props = "-Dexecution.checkpointing.savepoint-dir=hdfs:///test"; FlinkSavepointServiceImpl savepointServiceImpl = (FlinkSavepointServiceImpl) savepointService; @@ -93,30 +90,22 @@ void testGetSavepointFromDynamicProps() { } @Test - void testGetSavepointFromAppCfgIfStreamParkOrSQLJob() { + void readSavepointFromAppConfig() { FlinkSavepointServiceImpl savepointServiceImpl = (FlinkSavepointServiceImpl) savepointService; FlinkApplication app = new FlinkApplication(); Long appId = 1L; Long appCfgId = 1L; app.setId(appId); - // Test for non-(StreamPark job Or FlinkSQL job) app.setAppType(ApplicationType.APACHE_FLINK.getType()); assertThat(savepointServiceImpl.getSavepointFromConfig(app)).isNull(); - app.setAppType(ApplicationType.STREAMPARK_FLINK.getType()); - app.setJobType(FlinkJobType.FLINK_JAR.getMode()); - assertThat(savepointServiceImpl.getSavepointFromConfig(app)).isNull(); - // Test for (StreamPark job Or FlinkSQL job) without application config. - app.setAppType(ApplicationType.STREAMPARK_FLINK.getType()); - assertThat(savepointServiceImpl.getSavepointFromConfig(app)).isNull(); app.setAppType(ApplicationType.STREAMPARK_FLINK.getType()); app.setJobType(FlinkJobType.FLINK_JAR.getMode()); assertThat(savepointServiceImpl.getSavepointFromConfig(app)).isNull(); String ckDir = SAVEPOINT_DIRECTORY.key() + "=hdfs:///test"; - // Test for (StreamPark job Or FlinkSQL job) with application config just disabled checkpoint. FlinkApplicationConfig appCfg = new FlinkApplicationConfig(); appCfg.setId(appCfgId); appCfg.setAppId(appId); @@ -125,15 +114,6 @@ void testGetSavepointFromAppCfgIfStreamParkOrSQLJob() { configService.save(appCfg); assertThat(savepointServiceImpl.getSavepointFromConfig(app)).isNull(); - // Test for (StreamPark job or FlinkSQL job) with application config and enabled checkpoint and - // configured value. - - // Test for non-value for CHECKPOINTING_INTERVAL - appCfg.setContent(""); - configService.updateById(appCfg); - assertThat(savepointServiceImpl.getSavepointFromConfig(app)).isNull(); - - // Test for configured CHECKPOINTING_INTERVAL appCfg.setContent( DeflaterUtils.zipString( ckDir + "\n" @@ -150,7 +130,7 @@ void testGetSavepointFromAppCfgIfStreamParkOrSQLJob() { } @Test - void testGetSavepointFromDeployLayer() throws JsonProcessingException { + void readSavepointFromDeployConfig(@TempDir Path flinkHome) throws Exception { FlinkSavepointServiceImpl savepointServiceImpl = (FlinkSavepointServiceImpl) savepointService; Long appId = 1L; Long idOfFlinkEnv = 1L; @@ -164,30 +144,24 @@ void testGetSavepointFromDeployLayer() throws JsonProcessingException { FlinkEnv flinkEnv = new FlinkEnv(); flinkEnv.setFlinkName("mockFlinkName"); - flinkEnv.setFlinkHome("/tmp"); + flinkEnv.setFlinkHome(flinkHome.toString()); flinkEnv.setId(idOfFlinkEnv); flinkEnv.setVersion("1.15.3"); flinkEnv.setScalaVersion("2.12"); - flinkEnv.setFlinkConf(DeflaterUtils.zipString(SAVEPOINT_DIRECTORY.key() + ": hdfs:///test")); + Path confDir = Files.createDirectories(flinkHome.resolve("conf")); + Files.writeString( + confDir.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME), + SAVEPOINT_DIRECTORY.key() + ": hdfs:///test"); + FlinkEnvUtils.sync(flinkEnv); flinkEnvService.save(flinkEnv); - // Test for non-remote mode assertThat(savepointServiceImpl.getSavepointFromDeployLayer(application)) .isEqualTo("hdfs:///test"); - // Start the test lines for remote mode Long clusterId = 1L; - - // Test for it without cluster. application.setDeployMode(FlinkDeployMode.REMOTE.getMode()); application.setFlinkClusterId(clusterId); assertThatThrownBy(() -> savepointServiceImpl.getSavepointFromDeployLayer(application)) .isInstanceOf(NullPointerException.class); - - // Ignored. - // Test for it with empty config - // Test for it with the configured empty target value - // Test for it with the configured non-empty target value - } } diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkEnvUtilsTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkEnvUtilsTest.java new file mode 100644 index 0000000000..79ee2c76b4 --- /dev/null +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkEnvUtilsTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.util; + +import org.apache.streampark.common.util.FlinkConfigurationUtils; +import org.apache.streampark.console.core.entity.FlinkEnv; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies version-aware YAML handling at the console persistence boundary. */ +class FlinkEnvUtilsTest { + + @TempDir + Path flinkHome; + + @Test + void preserveFlink120YamlSelection() throws Exception { + Path configurationDirectory = Files.createDirectories(flinkHome.resolve("conf")); + Path legacyFile = + configurationDirectory.resolve( + FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME); + Files.writeString(legacyFile, "pipeline.name: legacy\n"); + Files.writeString( + configurationDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME), + "pipeline:\n name: standard\n"); + FlinkEnv environment = environment("1.20.0"); + + FlinkEnvUtils.sync(environment); + + assertThat(FlinkEnvUtils.isStandardYaml(environment)).isFalse(); + assertThat(FlinkEnvUtils.configuration(environment)) + .containsEntry("pipeline.name", "legacy"); + + Files.delete(legacyFile); + assertThat(FlinkEnvUtils.isStandardYaml(environment)).isFalse(); + assertThat(FlinkEnvUtils.configuration(environment)) + .containsEntry("pipeline.name", "legacy"); + + FlinkEnvUtils.sync(environment); + + assertThat(FlinkEnvUtils.isStandardYaml(environment)).isTrue(); + assertThat(FlinkEnvUtils.configuration(environment)) + .containsEntry("pipeline.name", "standard"); + } + + @Test + void refreshChangesYamlFormat() throws Exception { + Path legacyHome = createFlinkHome("flink-1.18", "1.18.2", "flink-conf.yaml"); + Path standardHome = createFlinkHome("flink-2.3", "2.3.0", "config.yaml"); + FlinkEnv environment = new FlinkEnv(); + + FlinkEnvUtils.refresh(environment, legacyHome.toString()); + assertThat(environment.getVersion()).isEqualTo("1.18.2"); + assertThat(FlinkEnvUtils.isStandardYaml(environment)).isFalse(); + + FlinkEnvUtils.refresh(environment, standardHome.toString()); + assertThat(environment.getVersion()).isEqualTo("2.3.0"); + assertThat(FlinkEnvUtils.isStandardYaml(environment)).isTrue(); + } + + private Path createFlinkHome(String directory, String version, String configFile) throws Exception { + Path home = Files.createDirectories(flinkHome.resolve(directory)); + Path lib = Files.createDirectories(home.resolve("lib")); + Path conf = Files.createDirectories(home.resolve("conf")); + Files.createFile(lib.resolve("flink-dist-" + version + ".jar")); + String yaml = + FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME.equals(configFile) + ? "parallelism.default: 1\n" + : "parallelism:\n default: 1\n"; + Files.writeString(conf.resolve(configFile), yaml); + return home; + } + + private FlinkEnv environment(String version) { + FlinkEnv environment = new FlinkEnv(); + environment.setFlinkHome(flinkHome.toString()); + environment.setVersion(version); + return environment; + } +} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/conf/ParameterCliTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilderTest.java similarity index 69% rename from streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/conf/ParameterCliTest.java rename to streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilderTest.java index dfc508cf6e..ebfa54ea7d 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/conf/ParameterCliTest.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilderTest.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.streampark.flink.core.conf; +package org.apache.streampark.console.core.util; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.FlinkOptions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -27,14 +27,14 @@ import static org.assertj.core.api.Assertions.assertThat; -class ParameterCliTest { +class FlinkShellCommandBuilderTest { @TempDir Path tempDir; @Test void readVmOptForNonUrlClassLoader() { - String result = ParameterCli.read(new String[]{"--vmopt"}); + String result = FlinkShellCommandBuilder.read(new String[]{"--vmopt"}); assertThat(result).contains("--add-opens"); } @@ -45,11 +45,11 @@ void readAppNameFromPropertiesFile() throws Exception { "app.properties"); Files.writeString( conf, - ConfigKeys.KEY_FLINK_PROPERTY_PREFIX() - + ConfigKeys.KEY_FLINK_APP_NAME() + FlinkOptions.PROPERTY_PREFIX.value() + + FlinkOptions.PIPELINE_NAME.key() + "=demo_app\n"); - String name = ParameterCli.read(new String[]{"--name", conf.toString()}); + String name = FlinkShellCommandBuilder.read(new String[]{"--name", conf.toString()}); assertThat(name).isEqualTo("demo_app"); } @@ -58,18 +58,19 @@ void readPropertyFromYamlFile() throws Exception { Path conf = tempDir.resolve("app.yml"); Files.writeString( conf, - ConfigKeys.KEY_FLINK_PROPERTY_PREFIX() - + ConfigKeys.KEY_FLINK_APP_NAME() + FlinkOptions.PROPERTY_PREFIX.value() + + FlinkOptions.PIPELINE_NAME.key() + ": demo_yaml\n"); - String property = ParameterCli.read(new String[]{"--property", conf.toString()}); - assertThat(property).contains("-D" + ConfigKeys.KEY_FLINK_APP_NAME() + "=demo_yaml"); + String property = + FlinkShellCommandBuilder.read(new String[]{"--property", conf.toString()}); + assertThat(property).contains("-D" + FlinkOptions.PIPELINE_NAME.key() + "=demo_yaml"); } @Test void emitWritesToProvidedStream() { java.io.ByteArrayOutputStream buffer = new java.io.ByteArrayOutputStream(); - ParameterCli.emit("hello", new java.io.PrintStream(buffer)); + FlinkShellCommandBuilder.emit("hello", new java.io.PrintStream(buffer)); assertThat(buffer.toString()).isEqualTo("hello"); } } diff --git a/streampark-console/streampark-console-webapp/.env b/streampark-console/streampark-console-webapp/.env index da2ab6ebd6..bee2c71c14 100644 --- a/streampark-console/streampark-console-webapp/.env +++ b/streampark-console/streampark-console-webapp/.env @@ -14,7 +14,7 @@ # limitations under the License. # port -VITE_PORT=10001 +VITE_PORT=9999 # spa-title VITE_GLOB_APP_TITLE="Apache StreamPark" diff --git a/streampark-console/streampark-console-webapp/.env.development b/streampark-console/streampark-console-webapp/.env.development index ea2a9e87ed..b10cddac8d 100644 --- a/streampark-console/streampark-console-webapp/.env.development +++ b/streampark-console/streampark-console-webapp/.env.development @@ -18,9 +18,9 @@ VITE_PUBLIC_PATH=/ # Cross-domain proxy, you can configure multiple # Please note that no line breaks -VITE_PROXY=[["/basic-api","http://localhost:10000"]] +VITE_PROXY=[["/basic-api","http://localhost:9999"]] -VITE_BASE_ADDRESS=http://localhost:10000 +VITE_BASE_ADDRESS=http://localhost:9999 # Delete console VITE_DROP_CONSOLE=false diff --git a/streampark-console/streampark-console-webapp/package.json b/streampark-console/streampark-console-webapp/package.json index 821f04336b..54378e0608 100644 --- a/streampark-console/streampark-console-webapp/package.json +++ b/streampark-console/streampark-console-webapp/package.json @@ -1,6 +1,6 @@ { "name": "streampark-webapp", - "version": "3.0.0-SNAPSHOT", + "version": "3.0.0", "author": { "name": "streampark", "url": "https://streampark.apache.org" diff --git a/streampark-console/streampark-console-webapp/src/views/flink/app/EditStreamPark.vue b/streampark-console/streampark-console-webapp/src/views/flink/app/EditStreamPark.vue index 2cc118ceb6..181bd908a5 100644 --- a/streampark-console/streampark-console-webapp/src/views/flink/app/EditStreamPark.vue +++ b/streampark-console/streampark-console-webapp/src/views/flink/app/EditStreamPark.vue @@ -295,6 +295,7 @@ isSetConfig = true; } const defaultFormValue = { isSetConfig, configOverride }; + const flinkSqlContent = res.flinkSql ? decodeByBase64(res.flinkSql) : ''; configOptions.forEach((item) => { Object.assign(defaultFormValue, { [item.key]: item.defaultValue, @@ -305,7 +306,7 @@ jobType: res.jobType, appType: res.appType, deployMode: res.deployMode, - flinkSql: res.flinkSql ? decodeByBase64(res.flinkSql) : '', + flinkSql: flinkSqlContent, dependency: '', module: res.module, configId, @@ -316,17 +317,16 @@ project: app.projectId, ...defaultFormValue, }); - nextTick(() => { - unref(flinkSql)?.setContent(decodeByBase64(res.flinkSql)); - - setTimeout(() => { - unref(dependencyRef)?.setDefaultValue(JSON.parse(res.dependency || '{}')); - unref(podTemplateRef)?.handleChoicePodTemplate('ptVisual', res.k8sPodTemplate); - unref(podTemplateRef)?.handleChoicePodTemplate('jmPtVisual', res.k8sJmPodTemplate); - unref(podTemplateRef)?.handleChoicePodTemplate('tmPtVisual', res.k8sTmPodTemplate); - }, 1000); - }); handleReset(); + await nextTick(); + unref(flinkSql)?.setContent(flinkSqlContent); + + setTimeout(() => { + unref(dependencyRef)?.setDefaultValue(JSON.parse(res.dependency || '{}')); + unref(podTemplateRef)?.handleChoicePodTemplate('ptVisual', res.k8sPodTemplate); + unref(podTemplateRef)?.handleChoicePodTemplate('jmPtVisual', res.k8sJmPodTemplate); + unref(podTemplateRef)?.handleChoicePodTemplate('tmPtVisual', res.k8sTmPodTemplate); + }, 1000); } function handleMergely(configOverride: string) { diff --git a/streampark-e2e/README.md b/streampark-e2e/README.md deleted file mode 100644 index 462a9aebfa..0000000000 --- a/streampark-e2e/README.md +++ /dev/null @@ -1,150 +0,0 @@ - -# StreamPark End-to-End Test - -## Page Object Model - -StreamPark End-to-End test respects -the [Page Object Model (POM)](https://www.selenium.dev/documentation/guidelines/page_object_models/) design pattern. -Every page of StreamPark is abstracted into a class for better maintainability. - -### Example - -The login page is abstracted -as [`LoginPage`](streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/LoginPage.java), with the -following fields, - -```java -public final class LoginPage { - @FindBy(id = "form_item_account") - private WebElement inputUsername; - - @FindBy(id = "form_item_password") - private WebElement inputPassword; - - @FindBy(xpath = "//button[contains(@classnames, 'login-button')]") - private WebElement buttonLogin; -} -``` - -where `inputUsername`, `inputPassword` and `buttonLogin` are the main elements on UI that we are interested in. They are -annotated with `FindBy` so that the test framework knows how to locate the elements on UI. You can locate the elements -by `id`, `className`, `css` selector, `tagName`, or even `xpath`, please refer -to [the JavaDoc](https://www.selenium.dev/selenium/docs/api/java/org/openqa/selenium/support/FindBy.html). - -**Note:** for better maintainability, it's essential to add some convenient `id` or `class` on UI for the wanted -elements if needed, avoid using too complex `xpath` selector or `css` selector that is not maintainable when UI have -styles changes. - -With those fields declared, we should also initialize them with a web driver. Here we pass the web driver into the -constructor and invoke `PageFactory.initElements` to initialize those fields, - -```java -public final class LoginPage { - // ... - public LoginPage(RemoteWebDriver driver) { - this.driver = driver; - - PageFactory.initElements(driver, this); - } -} -``` - -then, all those UI elements are properly filled in. - -## Test Environment Setup - -StreamPark End-to-End test uses [testcontainers](https://www.testcontainers.org) to set up the testing -environment, with docker compose. - -Typically, every test case needs one or more `docker-compose.yaml` files to set up all needed components, and expose the -StreamPark UI port for testing. You can use `@StreamPark(composeFiles = "")` and pass -the `docker-compose.yaml` files to automatically set up the environment in the test class. - -```java - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -class UserManagementTest { -} -``` - -You can get the web driver that is ready for testing in the class by adding a field of type `RemoteWebDriver`, which -will be automatically injected via the testing framework. - -```java - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -class UserManagementTest { - private RemoteWebDriver browser; -} -``` - -Then the field `browser` can be used in the test methods. - -```java - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -class UserManagementTest { - private RemoteWebDriver browser; - - @Test - @Order(10) - void testCreateUser() { - final UserManagementPage userManagementPage = new UserManagementPage(browser); - userManagementPage.createUser(newUserName, "test", password, newUserEmail, UserManagementUserType.ADMIN); - - Awaitility.await().untilAsserted(() -> assertThat(userManagementPage.userList()) - .as("User list should contain newly-created user") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newUserName))); - } -} -``` - -## Notes - -- For UI tests, it's common that the pages might need some time to load, or the operations might need some time to - complete, we can use `await().untilAsserted(() -> {})` to wait for the assertions. And use `new webDriverWait(driver, ExpectedConditions)` to wait for the elements to be present or clickable. -- For better maintainability, it's recommended to abstract the pages into classes, and use the Page Object Model design - pattern. - - -## Local development - -### Mac M1 -Add VM options to the test configuration in IntelliJ IDEA: -``` -# In this mode you need to install docker desktop for mac and run it with locally --Dm1_chip=true -``` - -### Running locally(without Docker) -``` -# In this mode you need to start frontend and backend services locally --Dlocal=true -``` - -### Running locally(with Docker) -``` -# In this mode you only need to install docker locally -``` - -- To run the tests locally, you need to have the StreamPark running locally. You should add `streampark-e2e/pom.xml` to the maven project -Since it does not participate in project compilation, it is not in the main project. -- Running run test class `org.apache.streampark.e2e.cases.UserManagementTest` in the IDE. After execution, the test video will be saved as mp4 in a local temporary directory. Such as -`/var/folders/hf/123/T/record-3123/PASSED-[engine:junit-jupiter]/[class:org.apache.streampark.e2e.cases.UserManagementTest]-20240606-152333.mp4` diff --git a/streampark-e2e/pom.xml b/streampark-e2e/pom.xml deleted file mode 100644 index a41604694d..0000000000 --- a/streampark-e2e/pom.xml +++ /dev/null @@ -1,141 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-e2e - pom - 1.0-SNAPSHOT - - - streampark-e2e-core - streampark-e2e-case - - - - 11 - 11 - UTF-8 - - 5.8.1 - 4.13.0 - 1.18.20 - 3.20.2 - 1.7.36 - 2.17.2 - 31.0.1-jre - - - - - - com.google.guava - guava - ${guava.version} - - - - org.junit - junit-bom - ${junit.version} - pom - import - - - - org.testcontainers - testcontainers-bom - 1.19.8 - pom - import - - - - - - - org.slf4j - slf4j-api - ${slf4j-api.version} - - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j-slf4j-impl.version} - - - - org.junit.jupiter - junit-jupiter - - - - org.testcontainers - testcontainers - - - - org.testcontainers - junit-jupiter - - - - org.testcontainers - selenium - - - - org.seleniumhq.selenium - selenium-chrome-driver - ${selenium.version} - - - - org.seleniumhq.selenium - selenium-support - ${selenium.version} - - - - org.assertj - assertj-core - ${assertj-core.version} - test - - - - org.projectlombok - lombok - ${lombok.version} - provided - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.22.2 - - - - diff --git a/streampark-e2e/streampark-e2e-case/pom.xml b/streampark-e2e/streampark-e2e-case/pom.xml deleted file mode 100644 index 5a0751bdb6..0000000000 --- a/streampark-e2e/streampark-e2e-case/pom.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-e2e - 1.0-SNAPSHOT - - - streampark-e2e-case - - - - org.apache.streampark - streampark-e2e-core - ${project.version} - - - diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/AlarmTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/AlarmTest.java deleted file mode 100644 index 87f1f4b17a..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/AlarmTest.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.setting.SettingPage; -import org.apache.streampark.e2e.pages.setting.alarm.AlarmPage; -import org.apache.streampark.e2e.pages.setting.alarm.AlertTypeDetailForm; -import org.apache.streampark.e2e.pages.setting.alarm.DingTalkAlertForm; -import org.apache.streampark.e2e.pages.setting.alarm.EmailAlertForm; -import org.apache.streampark.e2e.pages.setting.alarm.LarkAlertForm; -import org.apache.streampark.e2e.pages.setting.alarm.WeChatAlertForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -public class AlarmTest { - - public static RemoteWebDriver browser; - - private static final String newEmail = "new@streampark.com"; - - private static final String newAlarmName = "new_alarm"; - - private static final String editAlarmName = "edit_alarm"; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SettingPage.class) - .goToTab(AlarmPage.class); - } - - @Test - @Order(1) - public void testCreateAlarm() { - final AlarmPage alarmPage = new AlarmPage(browser); - - final String dingTalkURL = ""; - final String dingTalkToken = "dingTalkToken"; - final String dingTalkSecretToken = "dingTalkSecretToken"; - final String dingTalkReceiveUser = "dingTalkUser"; - - final String wechatToken = "wechatToken"; - - final String larkToken = "larkToken"; - final String larkSecretToken = "larkSecretToken"; - - AlertTypeDetailForm alertTypeDetailForm = alarmPage.createAlarm(); - alertTypeDetailForm - .addAlertType(AlertTypeDetailForm.AlertTypeEnum.EMAIL) - .email(newEmail) - .alertName(newAlarmName); - alertTypeDetailForm - .addAlertType(AlertTypeDetailForm.AlertTypeEnum.DINGTALK) - .url(dingTalkURL) - .token(dingTalkToken) - .secretEnable() - .secretToken(dingTalkSecretToken) - .effectToAllUsers() - .receiveUser(dingTalkReceiveUser); - alertTypeDetailForm - .addAlertType(AlertTypeDetailForm.AlertTypeEnum.WECHAT) - .token(wechatToken); - alertTypeDetailForm - .addAlertType(AlertTypeDetailForm.AlertTypeEnum.LARK) - .token(larkToken) - .secretEnable() - .secretToken(larkSecretToken) - .effectToAllUsers() - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(alarmPage.alarmList) - .as("Alarm list should contain newly-created alarm") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newAlarmName)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.EMAIL.desc)) - .anyMatch(it -> it.contains(newEmail)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.DINGTALK.desc)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.WECHAT.desc)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.LARK.desc)) - .anyMatch(it -> it.contains(dingTalkReceiveUser))); - } - - @Test - @Order(2) - public void testEditAlarm() { - final AlarmPage alarmPage = new AlarmPage(browser); - - alarmPage.editAlarm(newAlarmName) - // this step will cancel E-mail type click status. - .addAlertType(AlertTypeDetailForm.AlertTypeEnum.EMAIL) - .alertName(editAlarmName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(alarmPage.alarmList) - .as("Alarm list should contain edited alarm") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editAlarmName)) - .noneMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.EMAIL.desc)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.DINGTALK.desc)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.WECHAT.desc)) - .anyMatch(it -> it.contains(AlertTypeDetailForm.AlertTypeEnum.LARK.desc)) - .noneMatch(it -> it.contains(newEmail))); - } - - @Test - @Order(3) - public void testDeleteAlarm() { - final AlarmPage alarmPage = new AlarmPage(browser); - - alarmPage.deleteAlarm(editAlarmName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(alarmPage.alarmList) - .as(String.format("Alarm list shouldn't contain alarm witch named %s", editAlarmName)) - .extracting(WebElement::getText) - .noneMatch(it -> it.contains(editAlarmName))); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/EnvironmentTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/EnvironmentTest.java deleted file mode 100644 index bf1988167b..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/EnvironmentTest.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.setting.SettingPage; -import org.apache.streampark.e2e.pages.setting.env.DockerSettingForm; -import org.apache.streampark.e2e.pages.setting.env.EmailSettingForm; -import org.apache.streampark.e2e.pages.setting.env.EnvironmentDetailForm; -import org.apache.streampark.e2e.pages.setting.env.EnvironmentPage; -import org.apache.streampark.e2e.pages.setting.env.IngressSettingForm; -import org.apache.streampark.e2e.pages.setting.env.MavenSettingForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.ui.WebDriverWait; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementClick; -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/environment/docker-compose.yaml") -public class EnvironmentTest { - - public static RemoteWebDriver browser; - - // maven - final String mavenFilePath = "/maven/file/path"; - final String mavenCentralRepository = "https://mvnrepository.com/"; - final String mavenAuthUser = "maven_user"; - final String mavenAuthPassword = "maven_password"; - - // ingress - final String ingressDomainAddress = "https://localhost"; - - // docker - final String dockerAddress = "https://hub.docker.com/v2/"; - final String dockerNamespace = "hello"; - final String dockerUser = "docker_user"; - final String dockerPassword = "docker_password"; - - // email - final String emailHost = "smtp.163.com"; - final String editEmailHost = "postfix"; - final String emailPort = "25"; - final String emailAddress = "hello@163.com"; - final String editEmailAddress = "hello@postfix.com"; - final String emailUser = "email_password"; - final String emailPassword = "email_password"; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SettingPage.class) - .goToTab(EnvironmentPage.class); - } - - @Test - @Order(1) - public void testCreateEnvironment() { - final EnvironmentPage environmentPage = new EnvironmentPage(browser); - - environmentPage.createEnvironment(EnvironmentDetailForm.EnvSettingTypeEnum.Maven) - .addSetting(EnvironmentDetailForm.EnvSettingTypeEnum.Maven) - .filePath(mavenFilePath) - .centralRepository(mavenCentralRepository) - .authUser(mavenAuthUser) - .authPassword(mavenAuthPassword); - - environmentPage.createEnvironment(EnvironmentDetailForm.EnvSettingTypeEnum.Ingress) - .addSetting(EnvironmentDetailForm.EnvSettingTypeEnum.Ingress) - .domainAddress(ingressDomainAddress); - - Awaitility.await() - .untilAsserted( - () -> assertThat(environmentPage.settingList) - .as("Setting list should contain newly-created setting") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(mavenFilePath)) - .anyMatch(it -> it.contains(mavenCentralRepository)) - .anyMatch(it -> it.contains(mavenAuthUser)) - .anyMatch(it -> it.contains(ingressDomainAddress))); - } - - @Test - @Order(2) - public void testCreateEmailSettingFailedWithAuth() { - final EnvironmentPage environmentPage = new EnvironmentPage(browser); - - EmailSettingForm emailSettingForm = - environmentPage.createEnvironment(EnvironmentDetailForm.EnvSettingTypeEnum.Email) - .addSetting(EnvironmentDetailForm.EnvSettingTypeEnum.Email) - .host(emailHost) - .port(emailPort) - .address(emailAddress) - .user(emailUser) - .password(emailPassword) - .ok(); - - String expectedErrorMessage = - "connect to target mail server failed: 535 Error: authentication failed"; - Awaitility.await() - - .untilAsserted( - () -> { - new WebDriverWait(browser, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION); - assertThat(environmentPage.errorMessageList) - .as("Connect failed error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(expectedErrorMessage)); - }); - - WebElementClick(browser, environmentPage.errorMessageConfirmButton); - emailSettingForm.cancel(); - } - - @Test - @Order(3) - public void testCreateEmailSettingSuccessful() { - final EnvironmentPage environmentPage = new EnvironmentPage(browser); - - EmailSettingForm emailSettingForm = - environmentPage.createEnvironment(EnvironmentDetailForm.EnvSettingTypeEnum.Email) - .addSetting(EnvironmentDetailForm.EnvSettingTypeEnum.Email) - .host(editEmailHost) - .port(emailPort) - .address(editEmailAddress) - .user(emailUser) - .password(emailPassword) - .ok(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(environmentPage.settingList) - .as("Setting list should contain newly-created email setting") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editEmailAddress))); - } - - @Test - @Order(4) - public void testCreateDockerSettingFailed() { - final EnvironmentPage environmentPage = new EnvironmentPage(browser); - DockerSettingForm dockerSettingForm = - environmentPage.createEnvironment(EnvironmentDetailForm.EnvSettingTypeEnum.Docker) - .addSetting(EnvironmentDetailForm.EnvSettingTypeEnum.Docker) - .address(dockerAddress) - .namespace(dockerNamespace) - .user(dockerUser) - .password(dockerPassword) - .ok(); - - String expectedErrorMessage = String.format( - "Failed to validate Docker registry, error: Status 500: {\"message\":\"login attempt to %s failed with status: 404 Not Found\"}", - dockerAddress); - Awaitility.await() - - .untilAsserted( - () -> { - new WebDriverWait(browser, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION); - assertThat(environmentPage.errorMessageList) - .as("Failed to validate docker registry error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(expectedErrorMessage)); - }); - - WebElementClick(browser, environmentPage.errorMessageConfirmButton); - dockerSettingForm.cancel(); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ExternalLinkTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ExternalLinkTest.java deleted file mode 100644 index aba14b87be..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ExternalLinkTest.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.setting.ExternalLinkPage; -import org.apache.streampark.e2e.pages.setting.SettingPage; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -public class ExternalLinkTest { - - public static RemoteWebDriver browser; - - private static final String newLabel = "new_label"; - - private static final String editLabel = "edit_label"; - - private static final String newName = "new_name"; - - private static final String color = "#b54f4f"; - - private static final String newLink = "https://grafana/flink-monitoring?var-JobId=var-JobId=1"; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SettingPage.class) - .goToTab(ExternalLinkPage.class); - } - - @Test - @Order(1) - void testCreateExternalLink() { - final ExternalLinkPage externalLinkPage = new ExternalLinkPage(browser); - externalLinkPage.createExternalLink(newLabel, newName, color, newLink); - - Awaitility.await() - .untilAsserted( - () -> assertThat(externalLinkPage.externalLinkList) - .as("External link list should contain newly-created link") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newLabel)) - .anyMatch(it -> it.contains(newName)) - .anyMatch(it -> it.contains(newLink))); - } - - @Test - @Order(2) - void testCreateDuplicateExternalLink() { - final ExternalLinkPage externalLinkPage = new ExternalLinkPage(browser); - externalLinkPage.createExternalLink(newLabel, newName, color, newLink); - - Awaitility.await() - .untilAsserted( - () -> assertThat(externalLinkPage.errorMessageList) - .as("Name Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains( - String.format("The name: %s is already existing.", newName)))); - - externalLinkPage.errorMessageConfirmButton.click(); - externalLinkPage.createExternalLinkForm.buttonCancel.click(); - } - - @Test - @Order(3) - void testEditExternalLink() { - final ExternalLinkPage externalLinkPage = new ExternalLinkPage(browser); - String editName = "edit_name"; - String editLink = "https://grafana/flink-monitoring?var-Job=var-Job=edit"; - externalLinkPage.editExternalLink(newLabel, editLabel, editName, color, editLink); - - Awaitility.await() - .untilAsserted( - () -> assertThat(externalLinkPage.externalLinkList) - .as("External link list should contain edited link") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editLabel)) - .anyMatch(it -> it.contains(editName)) - .anyMatch(it -> it.contains(editLink))); - } - - @Test - @Order(4) - void testDeleteExternalLink() { - final ExternalLinkPage externalLinkPage = new ExternalLinkPage(browser); - - externalLinkPage.deleteExternalLink(editLabel); - - Awaitility.await() - .untilAsserted( - () -> { - assertThat(externalLinkPage.externalLinkList) - .noneMatch(it -> it.getText().contains(editLabel)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnRemoteClusterDeployTest.java deleted file mode 100644 index b8eb70bb0d..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnRemoteClusterDeployTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.RemoteForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.16-on-remote/docker-compose.yaml") -public class Flink116OnRemoteClusterDeployTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.16.3"; - - private static final String flinkHome = "/opt/flink/"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.16.3_cluster_e2e"; - - private static final String flinkJobManagerUrl = "http://jobmanager:8081"; - - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.STANDALONE; - - @BeforeAll - public static void setUp() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - } - - @Test - @Order(1) - public void testCreateFlinkCluster() { - FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .jobManagerURL(flinkJobManagerUrl) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); - } - - @Test - @Order(2) - public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnYarnClusterDeployTest.java deleted file mode 100644 index b48221907e..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink116OnYarnClusterDeployTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.16-on-yarn/docker-compose.yaml") -public class Flink116OnYarnClusterDeployTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.16.3"; - - private static final String flinkHome = "/flink-1.16.3"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.16.3_cluster_e2e"; - - private static final String flinkClusterNameEdited = "flink_1.16.3_cluster_e2e_edited"; - - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.YARN_SESSION; - - @BeforeAll - public static void setUp() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - } - - @Test - @Order(1) - public void testCreateFlinkCluster() { - FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); - } - - @Test - @Order(2) - public void testEditFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.editFlinkCluster(flinkClusterName) - .addCluster(deployMode) - .clusterName(flinkClusterNameEdited) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain edited application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterNameEdited))); - } - - @Test - @Order(3) - public void testStartFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.startFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain running application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - } - - @Test - @Order(4) - public void testStopFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.stopFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(5) - public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterNameEdited)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnRemoteClusterDeployTest.java deleted file mode 100644 index fdb0c5d019..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnRemoteClusterDeployTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.RemoteForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.17-on-remote/docker-compose.yaml") -public class Flink117OnRemoteClusterDeployTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.17.2"; - - private static final String flinkHome = "/opt/flink/"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.17.2_cluster_e2e"; - - private static final String flinkJobManagerUrl = "http://jobmanager:8081"; - - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.STANDALONE; - - @BeforeAll - public static void setUp() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - } - - @Test - @Order(1) - public void testCreateFlinkCluster() { - FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .jobManagerURL(flinkJobManagerUrl) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); - } - - @Test - @Order(5) - public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnYarnClusterDeployTest.java deleted file mode 100644 index 34e145f9ea..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink117OnYarnClusterDeployTest.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.17-on-yarn/docker-compose.yaml") -public class Flink117OnYarnClusterDeployTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.17.2"; - - private static final String flinkHome = "/flink-1.17.2"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.17.2_cluster_e2e"; - - private static final String flinkClusterNameEdited = "flink_1.17.2_cluster_e2e_edited"; - - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.YARN_SESSION; - - @BeforeAll - public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - } - - @Test - @Order(1) - public void testCreateFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .resolveOrder(YarnSessionForm.ResolveOrder.CHILD_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); - } - - @Test - @Order(2) - public void testEditFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.editFlinkCluster(flinkClusterName) - .addCluster(deployMode) - .clusterName(flinkClusterNameEdited) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain edited application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterNameEdited))); - } - - @Test - @Order(3) - public void testStartFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.startFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain running application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - } - - @Test - @Order(4) - public void testStopFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.stopFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(5) - public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterNameEdited)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnRemoteClusterDeployTest.java deleted file mode 100644 index 60f07c97a3..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnRemoteClusterDeployTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.RemoteForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.18-on-remote/docker-compose.yaml") -public class Flink118OnRemoteClusterDeployTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.18.1"; - - private static final String flinkHome = "/opt/flink/"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.18.1_cluster_e2e"; - - private static final String flinkJobManagerUrl = "http://jobmanager:8081"; - - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.STANDALONE; - - @BeforeAll - public static void setUp() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - } - - @Test - @Order(1) - public void testCreateFlinkCluster() { - FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .jobManagerURL(flinkJobManagerUrl) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); - } - - @Test - @Order(5) - public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnYarnClusterDeployTest.java deleted file mode 100644 index cadb7a39d2..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink118OnYarnClusterDeployTest.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.18-on-yarn/docker-compose.yaml") -public class Flink118OnYarnClusterDeployTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.18.1"; - - private static final String flinkHome = "/flink-1.18.1"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.18.1_cluster_e2e"; - - private static final String flinkClusterNameEdited = "flink_1.18.1_cluster_e2e_edited"; - - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.YARN_SESSION; - - @BeforeAll - public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - } - - @Test - @Order(1) - public void testCreateFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .resolveOrder(YarnSessionForm.ResolveOrder.CHILD_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); - } - - @Test - @Order(2) - public void testEditFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.editFlinkCluster(flinkClusterName) - .addCluster(deployMode) - .clusterName(flinkClusterNameEdited) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain edited application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterNameEdited))); - } - - @Test - @Order(3) - public void testStartFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.startFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain running application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - } - - @Test - @Order(4) - public void testStopFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.stopFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(5) - public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterNameEdited)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnRemoteClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnRemoteClusterDeployTest.java deleted file mode 100644 index 74c905a8ec..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnRemoteClusterDeployTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.RemoteForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.20-on-remote/docker-compose.yaml") -public class Flink120OnRemoteClusterDeployTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.20.1"; - - private static final String flinkHome = "/opt/flink/"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.20.1_cluster_e2e"; - - private static final String flinkJobManagerUrl = "http://jobmanager:8081"; - - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.STANDALONE; - - @BeforeAll - public static void setUp() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - } - - @Test - @Order(1) - public void testCreateFlinkCluster() { - FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .jobManagerURL(flinkJobManagerUrl) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); - } - - @Test - @Order(5) - public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnYarnClusterDeployTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnYarnClusterDeployTest.java deleted file mode 100644 index d6d379cc5d..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/Flink120OnYarnClusterDeployTest.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.20-on-yarn/docker-compose.yaml") -public class Flink120OnYarnClusterDeployTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.20.1"; - - private static final String flinkHome = "/flink-1.20.1"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.20.1_cluster_e2e"; - - private static final String flinkClusterNameEdited = "flink_1.20.1_cluster_e2e_edited"; - - private static final ClusterDetailForm.DeployMode deployMode = ClusterDetailForm.DeployMode.YARN_SESSION; - - @BeforeAll - public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - } - - @Test - @Order(1) - public void testCreateFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.createFlinkCluster() - .addCluster(deployMode) - .resolveOrder(YarnSessionForm.ResolveOrder.CHILD_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterName))); - } - - @Test - @Order(2) - public void testEditFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.editFlinkCluster(flinkClusterName) - .addCluster(deployMode) - .clusterName(flinkClusterNameEdited) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain edited application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkClusterNameEdited))); - } - - @Test - @Order(3) - public void testStartFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.startFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain running application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - } - - @Test - @Order(4) - public void testStopFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.stopFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkClustersPage.flinkClusterList) - .as("Flink clusters list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(5) - public void testDeleteFlinkCluster() { - final FlinkClustersPage flinkClustersPage = new FlinkClustersPage(browser); - - flinkClustersPage.deleteFlinkCluster(flinkClusterNameEdited); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - assertThat(flinkClustersPage.flinkClusterList) - .noneMatch(it -> it.getText().contains(flinkClusterNameEdited)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL116OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL116OnYarnTest.java deleted file mode 100644 index a08141a9a9..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL116OnYarnTest.java +++ /dev/null @@ -1,330 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationForm; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationsPage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; - -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.apache.streampark.e2e.pages.common.Constants.TEST_FLINK_SQL; -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.16-on-yarn/docker-compose.yaml") -public class FlinkSQL116OnYarnTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.16.3"; - - private static final String flinkHome = "/flink-1.16.3"; - - private static final String applicationName = "flink-116-e2e-test"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.16.3_cluster_e2e"; - - @BeforeAll - public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - FlinkClustersPage flinkClustersPage = flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - - flinkClustersPage.createFlinkCluster() - .addCluster(ClusterDetailForm.DeployMode.YARN_SESSION) - .resolveOrder(YarnSessionForm.ResolveOrder.PARENT_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - flinkClustersPage.startFlinkCluster(flinkClusterName); - - flinkClustersPage.goToNav(ApacheFlinkPage.class) - .goToTab(ApplicationsPage.class); - } - - @Test - @Order(1) - void testCreateFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_APPLICATION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(2) - void testReleaseFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(3) - void testStartFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(4) - @SneakyThrows - void testCancelFlinkApplicationOnYarnApplicationMode() { - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(5) - void testDeleteFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } - - @Test - @Order(6) - void testCreateFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_PER_JOB, - applicationName) - .flinkVersion(flinkName) - .flinkSql(TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(7) - void testReleaseFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(8) - void testStartFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain started application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(9) - void testDeleteFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } - - @Test - @Order(10) - void testCreateFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_SESSION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(TEST_FLINK_SQL) - .flinkCluster(flinkClusterName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(11) - void testReleaseFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(12) - void testStartFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain started application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(13) - void testDeleteFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL117OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL117OnYarnTest.java deleted file mode 100644 index c2cdaff273..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL117OnYarnTest.java +++ /dev/null @@ -1,357 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationForm; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationsPage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; - -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.17-on-yarn/docker-compose.yaml") -public class FlinkSQL117OnYarnTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.17.2"; - - private static final String flinkHome = "/flink-1.17.2"; - - private static final String applicationName = "flink-117-e2e-test"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.17.2_cluster_e2e"; - - @BeforeAll - public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - FlinkClustersPage flinkClustersPage = flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - - flinkClustersPage.createFlinkCluster() - .addCluster(ClusterDetailForm.DeployMode.YARN_SESSION) - .resolveOrder(YarnSessionForm.ResolveOrder.PARENT_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - flinkClustersPage.startFlinkCluster(flinkClusterName); - - flinkClustersPage.goToNav(ApacheFlinkPage.class) - .goToTab(ApplicationsPage.class); - } - - @Test - @Order(1) - void testCreateFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_APPLICATION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(2) - void testReleaseFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(3) - void testStartFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(4) - @SneakyThrows - void testCancelFlinkApplicationOnYarnApplicationMode() { - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(5) - void testDeleteFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } - - @Test - @Order(6) - void testCreateFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_PER_JOB, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(7) - void testReleaseFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(8) - void testStartFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain started application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(9) - void testDeleteFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } - - @Test - @Order(10) - void testCreateFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_SESSION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .flinkCluster(flinkClusterName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(11) - void testReleaseFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(12) - void testStartFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain started application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(13) - @SneakyThrows - void testRestartAndCancelFlinkApplicationOnYarnSessionMode() { - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(14) - void testDeleteFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL118OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL118OnYarnTest.java deleted file mode 100644 index df8e126cd9..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL118OnYarnTest.java +++ /dev/null @@ -1,371 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationForm; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationsPage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; - -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.18-on-yarn/docker-compose.yaml") -public class FlinkSQL118OnYarnTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.18.1"; - - private static final String flinkHome = "/flink-1.18.1"; - - private static final String applicationName = "flink-118-e2e-test"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.18.1_cluster_e2e"; - - @BeforeAll - public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - FlinkClustersPage flinkClustersPage = flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - - flinkClustersPage.createFlinkCluster() - .addCluster(ClusterDetailForm.DeployMode.YARN_SESSION) - .resolveOrder(YarnSessionForm.ResolveOrder.PARENT_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - flinkClustersPage.startFlinkCluster(flinkClusterName); - - flinkClustersPage.goToNav(ApacheFlinkPage.class) - .goToTab(ApplicationsPage.class); - } - - @Test - @Order(1) - void testCreateFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_APPLICATION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(2) - void testReleaseFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(3) - void testStartFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(4) - @SneakyThrows - void testCancelFlinkApplicationOnYarnApplicationMode() { - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(5) - void testDeleteFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } - - @Test - @Order(6) - void testCreateFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_PER_JOB, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(7) - void testReleaseFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(8) - void testStartFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain started application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(9) - @SneakyThrows - void testRestartAndCancelFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(10) - void testDeleteFlinkApplicationOnYarnPerJobMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } - - @Test - @Order(11) - void testCreateFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_SESSION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .flinkCluster(flinkClusterName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(12) - void testReleaseFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(13) - void testStartFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(14) - @SneakyThrows - void testRestartAndCancelFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(15) - void testDeleteFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - applicationsPage.deleteApplication(applicationName); - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL120OnYarnTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL120OnYarnTest.java deleted file mode 100644 index 39147c962c..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/FlinkSQL120OnYarnTest.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.flink.FlinkHomePage; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationForm; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationsPage; -import org.apache.streampark.e2e.pages.flink.clusters.ClusterDetailForm; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; -import org.apache.streampark.e2e.pages.flink.clusters.YarnSessionForm; - -import lombok.SneakyThrows; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/flink-1.20-on-yarn/docker-compose.yaml") -public class FlinkSQL120OnYarnTest { - - public static RemoteWebDriver browser; - - private static final String flinkName = "flink-1.20.1"; - - private static final String flinkHome = "/flink-1.20.1"; - - private static final String applicationName = "flink-120-e2e-test"; - - private static final String flinkDescription = "description test"; - - private static final String flinkClusterName = "flink_1.20.1_cluster_e2e"; - - @BeforeAll - public static void setup() { - FlinkHomePage flinkHomePage = new LoginPage(browser) - .login() - .goToNav(ApacheFlinkPage.class) - .goToTab(FlinkHomePage.class); - - flinkHomePage.createFlinkHome(flinkName, flinkHome, flinkDescription); - - FlinkClustersPage flinkClustersPage = flinkHomePage.goToNav(ApacheFlinkPage.class) - .goToTab(FlinkClustersPage.class); - - flinkClustersPage.createFlinkCluster() - .addCluster(ClusterDetailForm.DeployMode.YARN_SESSION) - .resolveOrder(YarnSessionForm.ResolveOrder.PARENT_FIRST) - .clusterName(flinkClusterName) - .flinkVersion(flinkName) - .submit(); - - flinkClustersPage.startFlinkCluster(flinkClusterName); - - flinkClustersPage.goToNav(ApacheFlinkPage.class) - .goToTab(ApplicationsPage.class); - } - - @Test - @Order(1) - void testCreateFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_APPLICATION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(2) - void testReleaseFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(3) - void testStartFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(4) - @SneakyThrows - void testCancelFlinkApplicationOnYarnApplicationMode() { - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(5) - void testDeleteFlinkApplicationOnYarnApplicationMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.deleteApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } - - @Test - @Order(11) - void testCreateFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage - .createApplication() - .addApplication( - ApplicationForm.FlinkJobType.FLINK_SQL, - ApplicationForm.DeployMode.YARN_SESSION, - applicationName) - .flinkVersion(flinkName) - .flinkSql(Constants.TEST_FLINK_SQL) - .flinkCluster(flinkClusterName) - .submit(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain newly-created application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(applicationName))); - } - - @Test - @Order(12) - void testReleaseFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.releaseApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain released application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESS"))); - } - - @Test - @Order(13) - void testStartFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain finished application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("FINISHED"))); - } - - @Test - @Order(14) - @SneakyThrows - void testRestartAndCancelFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - applicationsPage.startApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain restarted application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("RUNNING"))); - - applicationsPage.cancelApplication(applicationName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(applicationsPage.applicationsList) - .as("Applications list should contain canceled application") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("CANCELED"))); - } - - @Test - @Order(15) - void testDeleteFlinkApplicationOnYarnSessionMode() { - final ApplicationsPage applicationsPage = new ApplicationsPage(browser); - applicationsPage.deleteApplication(applicationName); - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(applicationsPage.applicationsList) - .noneMatch(it -> it.getText().contains(applicationName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/MemberManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/MemberManagementTest.java deleted file mode 100644 index db60907905..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/MemberManagementTest.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.system.MemberManagementPage; -import org.apache.streampark.e2e.pages.system.SystemPage; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -public class MemberManagementTest { - - public static RemoteWebDriver browser; - - private static final String existUserName = "test3"; - - private static final String existRole = "developer"; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SystemPage.class) - .goToTab(MemberManagementPage.class); - } - - @Test - @Order(1) - void testCreateMember() { - final MemberManagementPage memberManagementPage = new MemberManagementPage(browser); - - memberManagementPage.createMember(existUserName, existRole); - - Awaitility.await() - .untilAsserted( - () -> assertThat(memberManagementPage.memberList) - .as("Member list should contain newly-created member") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(existUserName))); - } - - @Test - @Order(2) - void testCreateDuplicateMember() { - final MemberManagementPage memberManagementPage = new MemberManagementPage(browser); - - memberManagementPage.createMember(existUserName, existRole); - - Awaitility.await() - .untilAsserted( - () -> assertThat(memberManagementPage.errorMessage) - .as("Member Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .matches(it -> it.contains("please don't add it again."))); - - memberManagementPage.errorMessageConfirmButton.click(); - memberManagementPage.createMemberForm.buttonCancel.click(); - } - - @Test - @Order(3) - void testEditMember() { - final MemberManagementPage memberManagementPage = new MemberManagementPage(browser); - String anotherRole = "team admin"; - - memberManagementPage.editMember(existUserName, anotherRole); - - Awaitility.await() - .untilAsserted( - () -> assertThat(memberManagementPage.memberList) - .as("Team list should contain edited team") - .extracting(WebElement::getText) - .anyMatch( - it -> it.contains(existUserName))); - } - - @Test - @Order(4) - void testDeleteMember() { - final MemberManagementPage memberManagementPage = new MemberManagementPage(browser); - - memberManagementPage.deleteMember(existUserName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(memberManagementPage.memberList) - .noneMatch(it -> it.getText().contains(existUserName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java deleted file mode 100644 index d6b1b82f60..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/ProjectsManagementTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.resource.ProjectsPage; -import org.apache.streampark.e2e.pages.resource.ResourcePage; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.PageFactory; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import java.time.Duration; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -public class ProjectsManagementTest { - - private final Duration PROJECT_BUILD_TIMEOUT_MINUTES = Duration.ofMinutes(15); - - public static RemoteWebDriver browser; - - private static final String projectName = "e2e_test_project"; - - private static final String editedProjectName = "e2e_test_project_edited"; - - private static final String url = "https://github.com/apache/streampark-quickstart"; - - private static final String branch = "dev"; - - private static final String buildArgument = - "-pl quickstart-flink/quickstart-apacheflink/apacheflinksql_1.17 -am -Dmaven.test.skip=true"; - - private static final String description = "e2e test project description"; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(ResourcePage.class) - .goToTab(ProjectsPage.class); - } - - @Test - @Order(1) - void testCreateProject() { - final ProjectsPage projectsPage = new ProjectsPage(browser); - - projectsPage.createProject(projectName, url, branch, buildArgument, description); - - Awaitility.await() - .untilAsserted( - () -> assertThat(projectsPage.projectList) - .as("Projects list should contain newly-created project") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(projectName))); - } - - @Test - @Order(2) - void testEditProject() { - final ProjectsPage projectsPage = new ProjectsPage(browser); - projectsPage.editProject(projectName, editedProjectName); - Awaitility.await() - .untilAsserted( - () -> assertThat(projectsPage.projectList) - .as("Projects list should contain edited project") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editedProjectName))); - } - - @Test - @Order(3) - void testBuildProject() { - ProjectsPage projectsPage = new ProjectsPage(browser); - - projectsPage.buildProject(editedProjectName); - - Awaitility.await().timeout(PROJECT_BUILD_TIMEOUT_MINUTES) - .untilAsserted( - () -> { - browser.navigate().refresh(); - PageFactory.initElements(projectsPage.driver, projectsPage); - assertThat(projectsPage.projectList) - .as("Projects list should contain build success project") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("SUCCESSFUL")); - }); - - } - - @Test - @Order(4) - void testDeleteProject() { - final ProjectsPage projectsPage = new ProjectsPage(browser); - projectsPage.deleteProject(editedProjectName); - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - assertThat(projectsPage.projectList) - .noneMatch(it -> it.getText().contains(editedProjectName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/RoleManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/RoleManagementTest.java deleted file mode 100644 index 18f7ace14a..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/RoleManagementTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.system.RoleManagementPage; -import org.apache.streampark.e2e.pages.system.SystemPage; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -public class RoleManagementTest { - - public static RemoteWebDriver browser; - - private static final String newRoleName = "new_role"; - - private static final String newDescription = "new_description"; - - private static final String existMenuName = "Apache Flink"; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SystemPage.class) - .goToTab(RoleManagementPage.class); - } - - @Test - @Order(1) - void testCreateRole() { - final RoleManagementPage roleManagementPage = new RoleManagementPage(browser); - roleManagementPage.createRole(newRoleName, newDescription, existMenuName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(roleManagementPage.roleList) - .as("Role list should contain newly-created role") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newRoleName))); - } - - @Test - @Order(2) - void testCreateDuplicateRole() { - final RoleManagementPage roleManagementPage = new RoleManagementPage(browser); - roleManagementPage.createRole(newRoleName, newDescription, existMenuName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(roleManagementPage.errorMessageList) - .as("Role Name Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains( - "Sorry, the role name already exists"))); - - roleManagementPage.createRoleForm.buttonCancel.click(); - } - @Test - @Order(3) - void testEditRole() { - final RoleManagementPage roleManagementPage = new RoleManagementPage(browser); - - String newEditDescription = newDescription + "_edit"; - String newEditMenuName = "System"; - roleManagementPage.editRole(newRoleName, newEditDescription, newEditMenuName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(roleManagementPage.roleList) - .as("Role list should contain edited role") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newEditDescription))); - } - - @Test - @Order(4) - void testDeleteRole() { - final RoleManagementPage roleManagementPage = new RoleManagementPage(browser); - - roleManagementPage.deleteRole(newRoleName); - - Awaitility.await() - .untilAsserted( - () -> assertThat(roleManagementPage.roleList) - .extracting(WebElement::getText) - .noneMatch(it -> it.contains(newRoleName))); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TeamManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TeamManagementTest.java deleted file mode 100644 index 331851b147..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TeamManagementTest.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.system.SystemPage; -import org.apache.streampark.e2e.pages.system.TeamManagementPage; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -public class TeamManagementTest { - - public static RemoteWebDriver browser; - - private static final String newTeamName = "test_new_team"; - - private static final String newTeamDescription = "test_new_team_description"; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SystemPage.class) - .goToTab(TeamManagementPage.class); - } - - @Test - @Order(1) - void testCreateTeam() { - final TeamManagementPage teamManagementPage = new TeamManagementPage(browser); - teamManagementPage.createTeam(newTeamName, newTeamDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(teamManagementPage.teamList) - .as("Team list should contain newly-created team") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newTeamName))); - } - - @Test - @Order(2) - void testCreateDuplicateTeam() { - final TeamManagementPage teamManagementPage = new TeamManagementPage(browser); - teamManagementPage.createTeam(newTeamName, newTeamDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(teamManagementPage.errorMessageList) - .as("Team Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("Create team failed."))); - - teamManagementPage.errorMessageConfirmButton.click(); - teamManagementPage.createTeamForm.buttonCancel.click(); - } - - @Test - @Order(3) - void testEditTeam() { - final TeamManagementPage teamManagementPage = new TeamManagementPage(browser); - String editDescription = "edit_" + newTeamDescription; - - teamManagementPage.editTeam(newTeamName, editDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(teamManagementPage.teamList) - .as("Team list should contain edited team") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editDescription))); - } - - @Test - @Order(4) - void testDeleteTeam() { - final TeamManagementPage teamManagementPage = new TeamManagementPage(browser); - - teamManagementPage.deleteTeam(newTeamName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(teamManagementPage.teamList) - .noneMatch(it -> it.getText().contains(newTeamName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TokenManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TokenManagementTest.java deleted file mode 100644 index 0d1bab7310..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/TokenManagementTest.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.system.SystemPage; -import org.apache.streampark.e2e.pages.system.TokenManagementPage; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.By; -import org.openqa.selenium.Keys; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -public class TokenManagementTest { - - public static RemoteWebDriver browser; - - private static final String existUserName = "admin"; - - private static final String newTokenDescription = "test_new_token_description"; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SystemPage.class) - .goToTab(TokenManagementPage.class); - } - - @Test - @Order(1) - void testCreateToken() { - final TokenManagementPage tokenManagementPage = new TokenManagementPage(browser); - tokenManagementPage.createToken(existUserName, newTokenDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(tokenManagementPage.tokenList) - .as("Token list should contain newly-created token") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(existUserName))); - } - - @Test - @Order(2) - void testCopyToken() { - final TokenManagementPage tokenManagementPage = new TokenManagementPage(browser); - tokenManagementPage.copyToken(existUserName); - - // put clipboard value into createTokenForm.description - tokenManagementPage.buttonCreateToken.click(); - tokenManagementPage.createTokenForm.inputDescription.sendKeys(Keys.CONTROL, "v"); - String token = tokenManagementPage.createTokenForm.inputDescription.getAttribute("value"); - tokenManagementPage.createTokenForm.buttonCancel.click(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(tokenManagementPage.tokenList) - .as("Clipboard should contain existing token.") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(token))); - } - - @Test - @Order(3) - void testCreateDuplicateToken() { - final TokenManagementPage tokenManagementPage = new TokenManagementPage(browser); - - tokenManagementPage.createToken(existUserName, newTokenDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(browser.findElement(By.tagName("body")).getText()) - .contains(String.format("user %s already has a token", existUserName))); - - tokenManagementPage.createTokenForm.buttonCancel.click(); - } - - @Test - @Order(4) - void testDeleteToken() { - final TokenManagementPage teamManagementPage = new TokenManagementPage(browser); - teamManagementPage.deleteToken(existUserName); - - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - assertThat(teamManagementPage.tokenList) - .noneMatch(it -> it.getText().contains(existUserName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UploadManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UploadManagementTest.java deleted file mode 100644 index 914b88e7b3..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UploadManagementTest.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.resource.ResourcePage; -import org.apache.streampark.e2e.pages.resource.UploadsPage; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -public class UploadManagementTest { - - public static RemoteWebDriver browser; - - private static final String engineType = "Apache Flink"; - - private static final String resourceType = "Jar Library"; - - private static final String resourceName = "test-resource"; - - private static final String mavenPom = - "\n" + - " junit\n" + - " junit\n" + - " 4.13.2\n" + - " test\n" + - ""; - - private static final String description = "Junit-jar-lib"; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(ResourcePage.class) - .goToTab(UploadsPage.class); - } - - @Test - @Order(1) - void testCreateUpload() { - final UploadsPage uploadsPage = new UploadsPage(browser); - uploadsPage.createUpload(engineType, resourceType, resourceName, mavenPom, description); - - Awaitility.await() - .untilAsserted( - () -> assertThat(uploadsPage.resourceList) - .as("Resource list should contain newly-created resource") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(resourceName)) - .anyMatch(it -> it.contains(description)) - .anyMatch(it -> it.contains(resourceType)) - .anyMatch(it -> it.contains(engineType))); - } - - @Test - @Order(2) - void testCreateDuplicateUpload() { - final UploadsPage uploadsPage = new UploadsPage(browser); - - uploadsPage.createUpload(engineType, resourceType, resourceName, mavenPom, description); - - Awaitility.await() - .untilAsserted( - () -> assertThat(uploadsPage.errorMessageList) - .as("Resource Name Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains( - String.format("the resource %s already exists, please check.", resourceName)))); - - uploadsPage.errorMessageConfirmButton.click(); - uploadsPage.createUploadForm.buttonCancel.click(); - } - - @Test - @Order(3) - void testEditUpload() { - final UploadsPage uploadsPage = new UploadsPage(browser); - browser.navigate().refresh(); - - String editDescription = "Kafka-jar-lib"; - String editResource = - "\n" + - " org.apache.kafka\n" + - " kafka-clients\n" + - " 3.7.1\n" + - ""; - - uploadsPage.editUpload(engineType, resourceType, resourceName, editResource, - editDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(uploadsPage.resourceList) - .as("Resource list should contain edit resource") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(resourceName)) - .anyMatch(it -> it.contains(editDescription)) - .anyMatch(it -> it.contains(resourceType)) - .anyMatch(it -> it.contains(engineType))); - } - - @Test - @Order(4) - void testDeleteUpload() { - final UploadsPage uploadsPage = new UploadsPage(browser); - uploadsPage.deleteUpload(resourceName); - Awaitility.await() - .untilAsserted( - () -> { - browser.navigate().refresh(); - - assertThat(uploadsPage.resourceList) - .noneMatch(it -> it.getText().contains(resourceName)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UserManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UserManagementTest.java deleted file mode 100644 index e41262db35..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/UserManagementTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.system.SystemPage; -import org.apache.streampark.e2e.pages.system.UserManagementPage; -import org.apache.streampark.e2e.pages.system.entity.UserManagementStatus; -import org.apache.streampark.e2e.pages.system.entity.UserManagementUserType; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -public class UserManagementTest { - - public static RemoteWebDriver browser; - - private static final String password = "streampark"; - - private static final String newUserName = "test_new"; - - private static final String newUserEmail = "test@email.com"; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SystemPage.class) - .goToTab(UserManagementPage.class); - } - - @Test - @Order(1) - void testCreateUser() { - final UserManagementPage userManagementPage = new UserManagementPage(browser); - userManagementPage.createUser( - newUserName, "test", password, newUserEmail, UserManagementUserType.ADMIN); - - Awaitility.await() - .untilAsserted( - () -> assertThat(userManagementPage.userList) - .as("User list should contain newly-created user") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newUserName))); - } - - @Test - @Order(2) - void testCreateDuplicateUser() { - final UserManagementPage userManagementPage = new UserManagementPage(browser); - userManagementPage.createUser( - newUserName, "test", password, "test@email.com", UserManagementUserType.ADMIN); - - Awaitility.await() - .untilAsserted( - () -> assertThat(userManagementPage.errorMessageList) - .as("User Name Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains( - "Sorry the username already exists"))); - - userManagementPage.createUserForm.buttonCancel.click(); - } - - @Test - @Order(3) - void testEditUser() { - final UserManagementPage userManagementPage = new UserManagementPage(browser); - String editEmail = "edit_" + newUserEmail; - - userManagementPage.editUser( - newUserName, editEmail, UserManagementUserType.ADMIN, UserManagementStatus.LOCKED); - - Awaitility.await() - .untilAsserted( - () -> assertThat(userManagementPage.userList) - .as("User list should contain edited user") - .extracting(WebElement::getText) - .anyMatch( - it -> it.contains(UserManagementStatus.LOCKED - .toString().toLowerCase()))); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/VariableManagementTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/VariableManagementTest.java deleted file mode 100644 index 0ad2fc5fad..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/VariableManagementTest.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.resource.ResourcePage; -import org.apache.streampark.e2e.pages.resource.VariablesPage; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -public class VariableManagementTest { - - public static RemoteWebDriver browser; - - private static final String variableCode = "10000"; - - private static final String variableValue = "3306"; - - private static final String description = "MySQL default port"; - - private static final boolean isNotVisible = true; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(ResourcePage.class) - .goToTab(VariablesPage.class); - } - - @Test - @Order(1) - void testCreateVariable() { - final VariablesPage variablesPage = new VariablesPage(browser); - variablesPage.createVariable(variableCode, variableValue, description, isNotVisible); - - Awaitility.await() - .untilAsserted( - () -> assertThat(variablesPage.variableList) - .as("Variable list should contain newly-created variable") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(variableCode))); - } - - @Test - @Order(2) - void testCreateDuplicateVariable() { - final VariablesPage variablesPage = new VariablesPage(browser); - variablesPage.createVariable(variableCode, variableValue, description, isNotVisible); - - Awaitility.await() - .untilAsserted( - () -> assertThat(variablesPage.errorMessageList) - .as("Variable Code Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains( - "The variable code already exists."))); - - variablesPage.errorMessageConfirmButton.click(); - variablesPage.createVariableForm.buttonCancel.click(); - } - - @Test - @Order(3) - void testEditVariable() { - final VariablesPage variablesPage = new VariablesPage(browser); - String editVariableValue = "6379"; - String editDescription = "Redis default port"; - - variablesPage.editVariable(variableCode, editVariableValue, editDescription, isNotVisible); - Awaitility.await() - .untilAsserted( - () -> assertThat(variablesPage.variableList) - .as("Variable list should contain edited variable") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editVariableValue)) - .anyMatch(it -> it.contains(editDescription))); - } - - @Test - @Order(4) - void testDeleteVariable() { - final VariablesPage variablesPage = new VariablesPage(browser); - - variablesPage.deleteVariable(variableCode); - - Awaitility.await() - .untilAsserted( - () -> assertThat(variablesPage.variableList) - .extracting(WebElement::getText) - .noneMatch(it -> it.contains(variableCode))); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/YarnQueueTest.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/YarnQueueTest.java deleted file mode 100644 index 25faeb6cb7..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/cases/YarnQueueTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.cases; - -import org.apache.streampark.e2e.core.StreamPark; -import org.apache.streampark.e2e.pages.LoginPage; -import org.apache.streampark.e2e.pages.setting.SettingPage; -import org.apache.streampark.e2e.pages.setting.YarnQueuePage; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import static org.assertj.core.api.Assertions.assertThat; - -@StreamPark(composeFiles = "docker/basic/docker-compose.yaml") -public class YarnQueueTest { - - public static RemoteWebDriver browser; - - private static final String newQueueLabel = "new_label"; - - private static final String editQueueLabel = "edit_label"; - - private static final String description = "test_description"; - - @BeforeAll - public static void setup() { - new LoginPage(browser) - .login() - .goToNav(SettingPage.class) - .goToTab(YarnQueuePage.class); - } - - @Test - @Order(1) - void testYarnQueue() { - final YarnQueuePage queuePage = new YarnQueuePage(browser); - queuePage.createYarnQueue(newQueueLabel, description); - - Awaitility.await() - .untilAsserted( - () -> assertThat(queuePage.yarnQueueList) - .as("Yarn Queue list should contain newly-created item") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(newQueueLabel)) - .anyMatch(it -> it.contains(description))); - } - - @Test - @Order(2) - void testCreateDuplicateYarnQueue() { - final YarnQueuePage queuePage = new YarnQueuePage(browser); - queuePage.createYarnQueue(newQueueLabel, description); - Awaitility.await() - .untilAsserted( - () -> assertThat(queuePage.errorMessageList) - .as("Yarn Queue Duplicated Error message should be displayed") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains("The queue label existed in the current team"))); - - queuePage.createYarnQueueForm.buttonCancel.click(); - } - - @Test - @Order(3) - void testEditYarnQueue() { - final YarnQueuePage queuePage = new YarnQueuePage(browser); - String editDescription = "edit_" + description; - - queuePage.editYarnQueue(newQueueLabel, editQueueLabel, editDescription); - - Awaitility.await() - .untilAsserted( - () -> assertThat(queuePage.yarnQueueList) - .as("Yarn queue list should contain edited yarn queue") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(editQueueLabel)) - .anyMatch(it -> it.contains(editDescription))); - } - - @Test - @Order(4) - void testDeleteYarnQueue() { - final YarnQueuePage queuePage = new YarnQueuePage(browser); - - queuePage.deleteYarnQueue(editQueueLabel); - Awaitility.await() - .untilAsserted( - () -> { - assertThat(queuePage.yarnQueueList) - .noneMatch(it -> it.getText().contains(editQueueLabel)); - }); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/LoginPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/LoginPage.java deleted file mode 100644 index a1c15c851c..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/LoginPage.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; - -import lombok.Getter; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -@Slf4j -public final class LoginPage extends NavBarPage { - - @FindBy(id = "form_item_account") - public WebElement inputUsername; - - @FindBy(id = "form_item_password") - public WebElement inputPassword; - - @FindBy(id = "e2e-login-btn") - public WebElement buttonLogin; - - private static final String userName = "admin"; - - private static final String password = "streampark"; - - public final TeamForm teamForm = new TeamForm(); - - public LoginPage(RemoteWebDriver driver) { - super(driver); - } - - @SneakyThrows - public NavBarPage login() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonLogin)); - inputUsername.sendKeys(userName); - inputPassword.sendKeys(password); - buttonLogin.click(); - return new NavBarPage(driver); - } - - @Getter - public class TeamForm { - - TeamForm() { - PageFactory.initElements(driver, this); - } - - @FindBys({ - @FindBy(css = "[popupClassName=team-select-popup]"), - @FindBy(className = "ant-select-item-option-content") - }) - public List selectTeam; - - @FindBy(css = "[popupClassName=team-select-popup] > .ant-select-selector") - public WebElement btnSelectTeamDropdown; - - @FindBy(xpath = "//button[contains(@class, 'ant-btn')]//span[contains(text(), 'OK')]") - public WebElement buttonSubmit; - - @FindBy(xpath = "//button[contains(@class, 'ant-btn')]//span[contains(text(), 'Cancel')]") - public WebElement buttonCancel; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java deleted file mode 100644 index 6c6995bcc3..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/CommonFactory.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.common; - -import org.openqa.selenium.By; -import org.openqa.selenium.ElementClickInterceptedException; -import org.openqa.selenium.JavascriptExecutor; -import org.openqa.selenium.Keys; -import org.openqa.selenium.Platform; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.time.Duration; - -public class CommonFactory { - - public static void WebElementDeleteAndInput(WebElement element, String value) { - WebElementDelete(element); - element.sendKeys(value); - } - - public static void WebElementDeleteAndInput(WebDriver driver, WebElement element, String value) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOf(element)); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(element)); - WebElementDelete(element); - element.sendKeys(value); - } - - public static void WebElementDelete(WebElement element) { - element.sendKeys(Keys.chord(selectAllKey(), "a")); - element.sendKeys(Keys.BACK_SPACE); - } - - public static void WebElementClick(WebDriver driver, WebElement clickableElement) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(clickableElement)); - try { - clickableElement.click(); - } catch (ElementClickInterceptedException ex) { - JavascriptExecutor executor = (JavascriptExecutor) driver; - executor.executeScript("arguments[0].scrollIntoView({block: 'center'});", clickableElement); - executor.executeScript("arguments[0].click();", clickableElement); - } - } - - private static CharSequence selectAllKey() { - return Platform.getCurrent().is(Platform.MAC) ? Keys.COMMAND : Keys.CONTROL; - } - - public static void WebDriverWaitForElementVisibilityAndInvisibility(WebDriver driver, String msg) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until( - ExpectedConditions.visibilityOfElementLocated( - By.xpath(String.format("//*[contains(text(),'%s')]", - msg)))); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until( - ExpectedConditions.invisibilityOfElementLocated( - By.xpath(String.format("//*[contains(text(),'%s')]", - msg)))); - } - - public static void WebDriverWaitForElementVisibilityAndInvisibilityWithDuration(WebDriver driver, String msg, - Duration duration) { - new WebDriverWait(driver, duration) - .until( - ExpectedConditions.visibilityOfElementLocated( - By.xpath(String.format("//*[contains(text(),'%s')]", - msg)))); - new WebDriverWait(driver, duration) - .until( - ExpectedConditions.invisibilityOfElementLocated( - By.xpath(String.format("//*[contains(text(),'%s')]", - msg)))); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/Constants.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/Constants.java deleted file mode 100644 index d02c20cb6a..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/Constants.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.common; - -import lombok.experimental.UtilityClass; - -import java.time.Duration; - -@UtilityClass -public class Constants { - - public static final Integer DEFAULT_SLEEP_MILLISECONDS = 2000; - - public static final Duration DEFAULT_WEBDRIVER_WAIT_DURATION = Duration.ofSeconds(10); - - public static final String LINE_SEPARATOR = "\n"; - - /** datagen flink sql for test */ - public static final String TEST_FLINK_SQL = "CREATE TABLE datagen (\n" - + "f_sequence INT,\n" - + "f_random INT,\n" - + "f_random_str STRING,\n" - + "ts AS localtimestamp,\n" - + "WATERMARK FOR ts AS ts\n" - + ") WITH (\n" - + "'connector' = 'datagen',\n" - + "'rows-per-second'='5',\n" - + "'fields.f_sequence.kind'='sequence',\n" - + "'fields.f_sequence.start'='1',\n" - + "'fields.f_sequence.end'='100',\n" - + "'fields.f_random.min'='1',\n" - + "'fields.f_random.max'='100',\n" - + "'fields.f_random_str.length'='10'\n" - + ");\n" - + "\n" - + "CREATE TABLE print_table (\n" - + "f_sequence INT,\n" - + "f_random INT,\n" - + "f_random_str STRING\n" - + ") WITH (\n" - + "'connector' = 'print'\n" - + ");\n" - + "\n" - + "INSERT INTO print_table select f_sequence,f_random,f_random_str from datagen;"; -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/NavBarPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/NavBarPage.java deleted file mode 100644 index aaa3b3bc13..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/common/NavBarPage.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.common; - -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; -import org.apache.streampark.e2e.pages.resource.ResourcePage; -import org.apache.streampark.e2e.pages.setting.SettingPage; -import org.apache.streampark.e2e.pages.system.SystemPage; - -import lombok.Getter; -import org.openqa.selenium.By; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -@Getter -public class NavBarPage { - - public final RemoteWebDriver driver; - - @FindBy(className = "menu-item-flink") - public WebElement apacheFlinkTab; - - @FindBy(className = "menu-item-resource") - public WebElement resourcesTab; - - @FindBy(className = "menu-item-setting") - public WebElement settingsTab; - - @FindBy(className = "menu-item-system") - public WebElement systemTab; - - public NavBarPage(RemoteWebDriver driver) { - this.driver = driver; - PageFactory.initElements(driver, this); - } - - public T goToNav(Class nav) { - if (nav == ApacheFlinkPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(apacheFlinkTab)); - String tabOpenStateXpath = - "//span[contains(@class, 'menu-item-flink')]/../parent::li[contains(@class, 'streampark-menu-opened')]"; - if (driver.findElements(By.xpath(tabOpenStateXpath)).isEmpty()) { - apacheFlinkTab.click(); - } - return nav.cast(new ApacheFlinkPage(driver)); - } - - if (nav == SystemPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(systemTab)); - String tabOpenStateXpath = - "//span[contains(@class, 'menu-item-system')]/../parent::li[contains(@class, 'streampark-menu-opened')]"; - if (driver.findElements(By.xpath(tabOpenStateXpath)).isEmpty()) { - systemTab.click(); - } - return nav.cast(new SystemPage(driver)); - } - - if (nav == ResourcePage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(resourcesTab)); - String tabOpenStateXpath = - "//span[contains(@class, 'menu-item-resource')]/../parent::li[contains(@class, 'streampark-menu-opened')]"; - if (driver.findElements(By.xpath(tabOpenStateXpath)).isEmpty()) { - resourcesTab.click(); - } - return nav.cast(new ResourcePage(driver)); - } - - if (nav == SettingPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(settingsTab)); - String tabOpenStateXpath = - "//span[contains(@class, 'menu-item-setting')]/../parent::li[contains(@class, 'streampark-menu-opened')]"; - if (driver.findElements(By.xpath(tabOpenStateXpath)).isEmpty()) { - settingsTab.click(); - } - return nav.cast(new SettingPage(driver)); - } - - throw new UnsupportedOperationException("Unknown nav bar"); - } - - public interface NavBarItem { - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/ApacheFlinkPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/ApacheFlinkPage.java deleted file mode 100644 index e94f31c59d..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/ApacheFlinkPage.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.flink; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; -import org.apache.streampark.e2e.pages.common.NavBarPage.NavBarItem; -import org.apache.streampark.e2e.pages.flink.applications.ApplicationsPage; -import org.apache.streampark.e2e.pages.flink.clusters.FlinkClustersPage; - -import lombok.Getter; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -@Getter -public final class ApacheFlinkPage extends NavBarPage implements NavBarItem { - - @FindBy(className = "menu-item-flink_app") - public WebElement menuApplications; - - @FindBy(className = "menu-item-flink_home") - public WebElement menuFlinkHome; - - @FindBy(className = "menu-item-flink_cluster") - public WebElement menuClusters; - - public ApacheFlinkPage(RemoteWebDriver driver) { - super(driver); - } - - public T goToTab(Class tab) { - if (tab == ApplicationsPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuApplications)); - menuApplications.click(); - return tab.cast(new ApplicationsPage(driver)); - } - - if (tab == FlinkHomePage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuFlinkHome)); - menuFlinkHome.click(); - return tab.cast(new FlinkHomePage(driver)); - } - - if (tab == FlinkClustersPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuClusters)); - menuClusters.click(); - return tab.cast(new FlinkClustersPage(driver)); - } - - throw new UnsupportedOperationException("Unknown tab: " + tab.getName()); - } - - public interface Tab { - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/FlinkHomePage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/FlinkHomePage.java deleted file mode 100644 index 8ce367e395..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/FlinkHomePage.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.flink; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; - -import lombok.Getter; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; -import org.testcontainers.shaded.org.awaitility.Awaitility; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -@Getter -public class FlinkHomePage extends NavBarPage implements ApacheFlinkPage.Tab { - - @FindBy(className = "ant-table-tbody") - public List flinkHomeList; - - @FindBy(id = "e2e-env-add-btn") - public WebElement buttonCreateFlinkHome; - - @FindBy(className = "e2e-flinkenv-delete-confirm") - public WebElement deleteConfirmButton; - - public CreateFlinkHomeForm createFlinkHomeForm; - - public FlinkHomePage(RemoteWebDriver driver) { - super(driver); - createFlinkHomeForm = new CreateFlinkHomeForm(); - } - - public FlinkHomePage createFlinkHome(String flinkName, String flinkHome, String description) { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateFlinkHome)); - - buttonCreateFlinkHome.click(); - createFlinkHomeForm.inputFlinkName.sendKeys(flinkName); - createFlinkHomeForm.inputFlinkHome.sendKeys(flinkHome); - createFlinkHomeForm.inputDescription.sendKeys(description); - createFlinkHomeForm.buttonSubmit.click(); - - Awaitility.await() - .untilAsserted( - () -> assertThat(flinkHomeList) - .as("FlinkEnv list should contain newly-created env") - .extracting(WebElement::getText) - .anyMatch(it -> it.contains(flinkName))); - - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/flink/home")); - } - - @Getter - public class CreateFlinkHomeForm { - - CreateFlinkHomeForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(id = "flink_env_flinkName") - public WebElement inputFlinkName; - - @FindBy(id = "flink_env_flinkHome") - public WebElement inputFlinkHome; - - @FindBy(id = "flink_env_description") - public WebElement inputDescription; - - @FindBy(id = "e2e-flinkenv-submit-btn") - public WebElement buttonSubmit; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationForm.java deleted file mode 100644 index b9f4ed1e7f..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationForm.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.flink.applications; - -import org.apache.streampark.e2e.pages.common.Constants; - -import lombok.Getter; -import lombok.SneakyThrows; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.interactions.Actions; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public final class ApplicationForm { - - public WebDriver driver; - - @FindBy(xpath = "//div[contains(@codefield, 'jobType')]//div[contains(@class, 'ant-select-selector')]") - public WebElement buttonJobTypeDropdown; - - @FindBys({ - @FindBy(css = "[codefield=jobType]"), - @FindBy(className = "ant-select-item-option-content") - }) - public List selectJobType; - - @FindBy(xpath = "//div[contains(@codefield, 'deployMode')]//div[contains(@class, 'ant-select-selector')]") - public WebElement buttonDeployModeDropdown; - - @FindBys({ - @FindBy(css = "[codefield=deployMode]"), - @FindBy(className = "ant-select-item-option-content") - }) - public List selectDeployMode; - - @FindBy(id = "form_item_jobName") - public WebElement inputApplicationName; - - @FindBy(xpath = "//div[contains(@codefield, 'yarnSessionClusterId')]//div[contains(@class, 'ant-select-selector')]") - public WebElement buttonFlinkClusterDropdown; - - @FindBy(className = "ant-select-item-option-content") - private List selectFlinkCluster; - - @FindBy(xpath = "//div[contains(@codefield, 'versionId')]//div[contains(@class, 'ant-select-selector')]") - public WebElement buttonFlinkVersionDropdown; - - @FindBys({ - @FindBy(css = "[codefield=versionId]"), - @FindBy(className = "ant-select-item-option-content") - }) - public List selectFlinkVersion; - - @FindBy(xpath = "//button[contains(@class, 'ant-btn')]//span[contains(text(), 'Submit')]") - public WebElement buttonSubmit; - - @FindBy(xpath = "//button[contains(@class, 'ant-btn')]//span[contains(text(), 'Cancel')]") - public WebElement buttonCancel; - - public ApplicationForm(WebDriver driver) { - this.driver = driver; - - PageFactory.initElements(driver, this); - } - - @SneakyThrows - public ApplicationForm addApplication(FlinkJobType jobType, - DeployMode deployMode, - String applicationName) { - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonJobTypeDropdown)); - buttonJobTypeDropdown.click(); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(selectJobType)); - switch (jobType) { - case FLINK_JAR: - selectJobType.stream() - .filter(e -> e.getText().equalsIgnoreCase(FlinkJobType.FLINK_JAR.desc)) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException( - String.format("Development mode not found: %s", - jobType.desc))) - .click(); - break; - case FLINK_SQL: - selectJobType.stream() - .filter(e -> e.getText().equalsIgnoreCase(FlinkJobType.FLINK_SQL.desc)) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException( - String.format("Development mode not found: %s", - jobType.desc))) - .click(); - buttonDeployModeDropdown.click(); - switch (deployMode) { - case REMOTE: - selectDeployMode.stream() - .filter(e -> e.getText().equalsIgnoreCase(DeployMode.REMOTE.desc)) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException( - String.format("Execution mode not found: %s", - deployMode.desc))) - .click(); - break; - case YARN_APPLICATION: - selectDeployMode.stream() - .filter(e -> e.getText() - .equalsIgnoreCase(DeployMode.YARN_APPLICATION.desc)) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException( - String.format("Execution mode not found: %s", - deployMode.desc))) - .click(); - - break; - case YARN_SESSION: - selectDeployMode.stream() - .filter(e -> e.getText().equalsIgnoreCase(DeployMode.YARN_SESSION.desc)) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException( - String.format("Execution mode not found: %s", - deployMode.desc))) - .click(); - break; - case KUBERNETES_SESSION: - selectDeployMode.stream() - .filter(e -> e.getText() - .equalsIgnoreCase(DeployMode.KUBERNETES_SESSION.desc)) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException( - String.format("Execution mode not found: %s", - deployMode.desc))) - .click(); - break; - case KUBERNETES_APPLICATION: - selectDeployMode.stream() - .filter( - e -> e.getText().equalsIgnoreCase( - DeployMode.KUBERNETES_APPLICATION.desc)) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException( - String.format("Execution mode not found: %s", - deployMode.desc))) - .click(); - break; - case YARN_PER_JOB: - selectDeployMode.stream() - .filter(e -> e.getText().equalsIgnoreCase(DeployMode.YARN_PER_JOB.desc)) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException( - String.format("Execution mode not found: %s", - deployMode.desc))) - .click(); - break; - default: - throw new IllegalArgumentException( - String.format("Unknown execution mode: %s", deployMode.desc)); - } - break; - case PYTHON_FLINK: - selectJobType.stream() - .filter(e -> e.getText().equalsIgnoreCase(FlinkJobType.PYTHON_FLINK.toString())) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException( - String.format("Development mode not found: %s", - jobType))) - .click(); - break; - default: - throw new IllegalArgumentException( - String.format("Unknown development mode: %s", jobType)); - } - inputApplicationName.sendKeys(applicationName); - - return this; - } - - public ApplicationForm flinkVersion(String flinkVersion) { - new Actions(driver).moveToElement(buttonFlinkVersionDropdown).build().perform(); - buttonFlinkVersionDropdown.click(); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(selectFlinkVersion)); - selectFlinkVersion.stream() - .filter(e -> e.getText().equalsIgnoreCase(flinkVersion)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException("Flink version not found")) - .click(); - - return this; - } - - public ApplicationForm flinkSql(String flinkSql) { - new FlinkSQLEditor(driver).content(flinkSql); - return this; - } - - @SneakyThrows - public ApplicationForm flinkCluster(String flinkClusterName) { - new Actions(driver).moveToElement(buttonFlinkClusterDropdown).build().perform(); - buttonFlinkClusterDropdown.click(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - selectFlinkCluster.stream() - .filter(e -> e.getText().contains(flinkClusterName)) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException( - String.format("Flink cluster not found: %s", flinkClusterName))) - .click(); - - return this; - } - - public ApplicationForm submit() { - buttonSubmit.click(); - - return this; - } - - @Getter - public enum FlinkJobType { - - FLINK_JAR("flink jar"), - FLINK_SQL("flink sql"), - PYTHON_FLINK("python flink"); - - private final String desc; - - FlinkJobType(String desc) { - this.desc = desc; - } - } - - @Getter - public enum DeployMode { - - REMOTE("remote"), - YARN_APPLICATION("yarn application"), - YARN_SESSION("yarn session"), - KUBERNETES_SESSION("kubernetes session"), - KUBERNETES_APPLICATION("kubernetes application"), - YARN_PER_JOB("yarn per-job (deprecated, please use yarn-application mode)"); - - private final String desc; - - DeployMode(String desc) { - this.desc = desc; - } - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationsPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationsPage.java deleted file mode 100644 index 3632caca21..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/ApplicationsPage.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.flink.applications; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; - -import lombok.Getter; -import org.openqa.selenium.By; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.interactions.Actions; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class ApplicationsPage extends NavBarPage implements ApacheFlinkPage.Tab { - - @FindBy(id = "e2e-flinkapp-create-btn") - public WebElement buttonCreateApplication; - - @FindBy(className = "ant-table-tbody") - public List applicationsList; - - @FindBy(className = "ant-form-item-explain-error") - public List errorMessageList; - - @FindBy(className = "e2e-flinkapp-delete-btn") - public WebElement deleteButton; - - @FindBy(className = "e2e-flinkapp-delete-confirm") - public WebElement deleteConfirmButton; - - public ApplicationsPage(RemoteWebDriver driver) { - super(driver); - } - - public ApplicationForm createApplication() { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateApplication)); - - buttonCreateApplication.click(); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/flink/app/add")); - return new ApplicationForm(driver); - } - - public ApplicationsPage deleteApplication(String applicationName) { - waitForPageLoading(); - - WebElement extraButton = applicationsList.stream() - .filter(it -> it.getText().contains(applicationName)) - .flatMap( - it -> it.findElements(By.className("anticon-more")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No extra button in applications list")); - Actions actions = new Actions(this.driver); - actions.moveToElement(extraButton).perform(); - - deleteButton.click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); - - deleteConfirmButton.click(); - - return this; - } - - public ApplicationsPage startApplication(String applicationName) { - waitForPageLoading(); - - applicationsList.stream() - .filter(it -> it.getText().contains(applicationName)) - .flatMap(it -> it.findElements(By.className("e2e-flinkapp-startup-btn")).stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No start button in applications list")) - .click(); - - StartJobForm startJobForm = new StartJobForm(); - String startJobFormMessage = "Start Job"; - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until( - ExpectedConditions.visibilityOfElementLocated( - By.xpath(String.format("//*[contains(.,'%s')]", - startJobFormMessage)))); - - startJobForm.buttonSubmit.click(); - return this; - } - - public ApplicationsPage releaseApplication(String applicationName) { - waitForPageLoading(); - - applicationsList.stream() - .filter(it -> it.getText().contains(applicationName)) - .flatMap( - it -> it.findElements(By.className("e2e-flinkapp-release-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No release button in applications list")) - .click(); - - return this; - } - - public ApplicationsPage cancelApplication(String applicationName) { - waitForPageLoading(); - - applicationsList.stream() - .filter(it -> it.getText().contains(applicationName)) - .flatMap( - it -> it.findElements(By.className("e2e-flinkapp-cancel-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No cancel button in applications list")) - .click(); - - CancelJobForm cancelJobForm = new CancelJobForm(); - String cancelJobFormMessage = "Stop Job"; - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until( - ExpectedConditions.visibilityOfElementLocated( - By.xpath(String.format("//*[contains(.,'%s')]", - cancelJobFormMessage)))); - - cancelJobForm.buttonSubmit.click(); - - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/flink/app")); - } - - @Getter - public class StartJobForm { - - StartJobForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(id = "e2e-flinkapp-start-submit") - public WebElement buttonSubmit; - } - - @Getter - public class CancelJobForm { - - CancelJobForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(id = "e2e-flinkapp-stop-submit") - public WebElement buttonSubmit; - - @FindBy(id = "e2e-flinkapp-stop-cancel") - public WebElement buttonCancel; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/FlinkSQLEditor.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/FlinkSQLEditor.java deleted file mode 100644 index 05c197db1f..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/applications/FlinkSQLEditor.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.flink.applications; - -import org.apache.streampark.e2e.pages.common.Constants; - -import lombok.Getter; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import org.junit.platform.commons.util.StringUtils; -import org.openqa.selenium.Keys; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.interactions.Actions; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.Arrays; -import java.util.List; - -@Getter -@Slf4j -public final class FlinkSQLEditor { - - @FindBy(xpath = "//label[contains(@for, 'form_item_flinkSql')]/../..//div[contains(@class, 'monaco-editor')]//div[contains(@class, 'view-line') and not(contains(@class, 'view-lines'))]") - public List flinkSqlEditor; - - public WebDriver driver; - - public static final Integer FLINK_SQL_EDITOR_SLEEP_MILLISECONDS = 1000; - - public FlinkSQLEditor(WebDriver driver) { - PageFactory.initElements(driver, this); - this.driver = driver; - } - - @SneakyThrows - public FlinkSQLEditor content(String content) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(flinkSqlEditor.get(0))); - - Actions actions = new Actions(this.driver); - - List contentList = Arrays.asList(content.split(Constants.LINE_SEPARATOR)); - - for (int i = 0; i < contentList.size(); i++) { - String editorLineText; - String inputContent = contentList.get(i); - int flinkSqlEditorIndex = Math.min(i, 21); - - if (i == 0) { - actions.moveToElement(flinkSqlEditor.get(flinkSqlEditorIndex)) - .click() - .sendKeys(inputContent) - .sendKeys(Constants.LINE_SEPARATOR) - .perform(); - continue; - } else { - editorLineText = flinkSqlEditor.get(flinkSqlEditorIndex).getText(); - } - - if (StringUtils.isNotBlank(inputContent)) { - if (!editorLineText.isEmpty()) { - for (int p = 0; p < editorLineText.trim().length(); p++) { - clearLine(actions, flinkSqlEditor.get(flinkSqlEditorIndex)); - } - clearLine(actions, flinkSqlEditor.get(flinkSqlEditorIndex)); - } - actions.moveToElement(flinkSqlEditor.get(flinkSqlEditorIndex)) - .click() - .sendKeys(inputContent) - .sendKeys(Constants.LINE_SEPARATOR) - .perform(); - Thread.sleep(FLINK_SQL_EDITOR_SLEEP_MILLISECONDS); - } else { - actions.moveToElement(flinkSqlEditor.get(flinkSqlEditorIndex)) - .click() - .sendKeys(Constants.LINE_SEPARATOR) - .perform(); - Thread.sleep(FLINK_SQL_EDITOR_SLEEP_MILLISECONDS); - } - } - - return this; - } - - private void clearLine(Actions actions, WebElement element) { - actions.moveToElement(element) - .click() - .sendKeys(Keys.BACK_SPACE) - .perform(); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/ClusterDetailForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/ClusterDetailForm.java deleted file mode 100644 index c483ee8b62..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/ClusterDetailForm.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.flink.clusters; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; -import org.openqa.selenium.support.PageFactory; - -import java.util.List; - -@Getter -public class ClusterDetailForm { - - public WebDriver driver; - - @FindBy(xpath = "//div[contains(@codefield, 'deployMode')]//div[contains(@class, 'ant-select-selector')]") - public WebElement buttonDeployModeDropdown; - - @FindBys({ - @FindBy(css = "[codefield=deployMode]"), - @FindBy(className = "ant-select-item-option-content") - }) - private List selectDeployMode; - - public ClusterDetailForm(WebDriver driver) { - PageFactory.initElements(driver, this); - this.driver = driver; - } - - @SuppressWarnings("unchecked") - public T addCluster(DeployMode deployMode) { - buttonDeployModeDropdown.click(); - switch (deployMode) { - case STANDALONE: - selectDeployMode.stream() - .filter(e -> e.getText().equalsIgnoreCase(DeployMode.STANDALONE.desc)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - String.format("Execution Mode not found: %s", deployMode.desc))) - .click(); - return (T) new RemoteForm(this); - case YARN_SESSION: - selectDeployMode.stream() - .filter(e -> e.getText().equalsIgnoreCase(DeployMode.YARN_SESSION.desc)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException( - String.format("Execution Mode not found: %s", deployMode.desc))) - .click(); - return (T) new YarnSessionForm(this); - default: - throw new UnsupportedOperationException( - String.format("Unknown execution mode: %s", deployMode.desc)); - } - } - - @Getter - public enum DeployMode { - - STANDALONE("standalone"), - YARN_SESSION("yarn session"), - KUBERNETES_SESSION( - "kubernetes session"); - - private final String desc; - - DeployMode(String desc) { - this.desc = desc; - } - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/CommonForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/CommonForm.java deleted file mode 100644 index da6751e3d3..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/CommonForm.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.flink.clusters; - -import org.apache.streampark.e2e.pages.common.Constants; - -import lombok.Getter; -import lombok.SneakyThrows; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; -import org.openqa.selenium.support.PageFactory; - -import java.util.List; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; - -@Getter -public abstract class CommonForm { - - public WebDriver driver; - - @FindBy(id = "flink_cluster_clusterName") - public WebElement inputFlinkClusterName; - - @FindBy(xpath = "//div[contains(@codefield, 'versionId')]//div[contains(@class, 'ant-select-selector')]") - public WebElement buttonFlinkVersionDropdown; - - @FindBys({ - @FindBy(css = "[codefield=versionId]"), - @FindBy(className = "ant-select-item-option-content") - }) - public List selectFlinkVersion; - - @FindBy(id = "e2e-flinkcluster-submit-btn") - public WebElement buttonSubmit; - - private final ClusterDetailForm parent; - - CommonForm(ClusterDetailForm clusterDetailForm) { - final WebDriver driver = clusterDetailForm.driver; - - PageFactory.initElements(driver, this); - - this.parent = clusterDetailForm; - } - - public CommonForm clusterName(String clusterName) { - WebElementDeleteAndInput(inputFlinkClusterName, clusterName); - return this; - } - - @SneakyThrows - public CommonForm flinkVersion(String flinkVersion) { - buttonFlinkVersionDropdown.click(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - selectFlinkVersion.stream() - .filter(e -> e.getText().equalsIgnoreCase(flinkVersion)) - .findFirst() - .orElseThrow(() -> new IllegalArgumentException(String.format("Flink version not found: %s", flinkVersion))) - .click(); - - return this; - } - - public ClusterDetailForm submit() { - buttonSubmit.click(); - return parent; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/FlinkClustersPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/FlinkClustersPage.java deleted file mode 100644 index b18983d4e4..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/FlinkClustersPage.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.flink.clusters; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; -import org.apache.streampark.e2e.pages.flink.ApacheFlinkPage; - -import lombok.Getter; -import lombok.SneakyThrows; -import org.openqa.selenium.By; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class FlinkClustersPage extends NavBarPage implements ApacheFlinkPage.Tab { - - @FindBy(id = "e2e-flinkcluster-create-btn") - public WebElement buttonCreateFlinkCluster; - - @FindBy(className = "ant-table-tbody") - public List flinkClusterList; - - @FindBy(className = "e2e-flinkcluster-delete-confirm") - public WebElement deleteConfirmButton; - - public FlinkClustersPage(RemoteWebDriver driver) { - super(driver); - } - - public ClusterDetailForm createFlinkCluster() { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateFlinkCluster)); - buttonCreateFlinkCluster.click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/flink/add_cluster")); - - return new ClusterDetailForm(driver); - } - - public ClusterDetailForm editFlinkCluster(String flinkClusterName) { - waitForPageLoading(); - - flinkClusterList.stream() - .filter(it -> it.getText().contains(flinkClusterName)) - .flatMap( - it -> it.findElements( - By.className("e2e-flinkcluster-edit-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in flink clusters list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/flink/edit_cluster")); - - return new ClusterDetailForm(driver); - } - - @SneakyThrows - public FlinkClustersPage startFlinkCluster(String flinkClusterName) { - waitForPageLoading(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - flinkClusterList.stream() - .filter(it -> it.getText().contains(flinkClusterName)) - .flatMap( - it -> it - .findElements( - By.className("e2e-flinkcluster-start-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No start button in flink clusters list")) - .click(); - - return this; - } - - public FlinkClustersPage stopFlinkCluster(String flinkClusterName) { - waitForPageLoading(); - - flinkClusterList.stream() - .filter(it -> it.getText().contains(flinkClusterName)) - .flatMap( - it -> it - .findElements( - By.className("e2e-flinkcluster-shutdown-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No stop button in flink clusters list")) - .click(); - - return this; - } - - public FlinkClustersPage deleteFlinkCluster(String flinkName) { - waitForPageLoading(); - - flinkClusterList.stream() - .filter(it -> it.getText().contains(flinkName)) - .flatMap( - it -> it - .findElements( - By.className("e2e-flinkcluster-delete-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in flink clusters list")) - .click(); - - deleteConfirmButton.click(); - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/flink/cluster")); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/RemoteForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/RemoteForm.java deleted file mode 100644 index 51332964d2..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/RemoteForm.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.flink.clusters; - -import org.apache.streampark.e2e.pages.common.CommonFactory; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; - -@Getter -public class RemoteForm extends CommonForm { - - public WebDriver driver; - - @FindBy(id = "flink_cluster_address") - public WebElement inputJobManagerURL; - - public RemoteForm(ClusterDetailForm clusterDetailForm) { - super(clusterDetailForm); - this.driver = clusterDetailForm.driver; - } - - public RemoteForm jobManagerURL(String address) { - CommonFactory.WebElementDeleteAndInput(this.inputJobManagerURL, address); - return this; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/YarnSessionForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/YarnSessionForm.java deleted file mode 100644 index 32e5058596..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/flink/clusters/YarnSessionForm.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.flink.clusters; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; - -import java.util.List; - -@Getter -public class YarnSessionForm extends CommonForm { - - public WebDriver driver; - - @FindBy(xpath = "//div[contains(@codefield, 'resolveOrder')]//div[contains(@class, 'ant-select-selector')]") - public WebElement buttonResolveOrderDropdown; - - @FindBys({ - @FindBy(css = "[codefield=resolveOrder]"), - @FindBy(className = "ant-select-item-option-content") - }) - private List selectResolveOrder; - - public YarnSessionForm(ClusterDetailForm clusterDetailForm) { - super(clusterDetailForm); - this.driver = clusterDetailForm.driver; - } - - public YarnSessionForm resolveOrder(ResolveOrder resolveOrder) { - buttonResolveOrderDropdown.click(); - selectResolveOrder.stream() - .filter(e -> e.getText().equalsIgnoreCase(resolveOrder.desc)) - .findFirst() - .orElseThrow( - () -> new IllegalArgumentException(String.format("Resolve Order not found: %s", resolveOrder.desc))) - .click(); - - return this; - } - - @Getter - public enum ResolveOrder { - - PARENT_FIRST("parent-first"), - CHILD_FIRST("child-first"); - - private final String desc; - - ResolveOrder(String desc) { - this.desc = desc; - } - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java deleted file mode 100644 index 3fbe135957..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ProjectsPage.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.resource; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; - -import lombok.Getter; -import lombok.SneakyThrows; -import org.openqa.selenium.By; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementClick; -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; - -@Getter -public class ProjectsPage extends NavBarPage implements ResourcePage.Tab { - - @FindBy(id = "e2e-project-create-btn") - public WebElement buttonCreateProject; - - @FindBy(className = "ant-table-tbody") - public List projectList; - - @FindBy(className = "e2e-project-build-btn") - public WebElement buildButton; - - @FindBy(className = "e2e-project-delete-confirm") - public WebElement deleteConfirmButton; - - @FindBy(className = "e2e-project-build-confirm") - public WebElement buildConfirmButton; - - public CreateProjectForm createProjectForm; - - public ProjectsPage(RemoteWebDriver driver) { - super(driver); - createProjectForm = new CreateProjectForm(); - } - - @SneakyThrows - public ProjectsPage createProject(String projectName, - String projectUrl, - String projectRefs, - String projectBuildArgument, - String projectDescription) { - waitForPageLoading(); - - WebElementClick(driver, buttonCreateProject); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/project/add")); - - createProjectForm = new CreateProjectForm(); - createProjectForm.inputProjectName.sendKeys(projectName); - - createProjectForm.selectCveDropdown.click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(d -> createProjectForm.selectCve.stream().anyMatch(WebElement::isDisplayed)); - - createProjectForm.selectCve.stream() - .filter(e -> e.getText().equalsIgnoreCase("GitHub/GitLab")) - .findFirst() - .orElseThrow(() -> new Exception("Cvs not found")) - .click(); - - createProjectForm.inputProjectUrl.sendKeys(projectUrl); - createProjectForm.selectBranchDropdown.click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(d -> createProjectForm.selectRefs.stream().anyMatch(WebElement::isDisplayed)); - createProjectForm.selectRefs.stream() - .filter(e -> e.getText().equalsIgnoreCase(projectRefs)) - .findFirst() - .orElseThrow(() -> new Exception(String.format("Branch not found: %s", projectRefs))) - .click(); - - createProjectForm.inputBuildArgument.sendKeys(projectBuildArgument); - createProjectForm.inputDescription.sendKeys(projectDescription); - createProjectForm.buttonSubmit.click(); - - waitForListPageAfterSubmit(); - return this; - } - - @SneakyThrows - public ProjectsPage editProject(String projectName, - String newProjectName) { - waitForPageLoading(); - - WebElement editButton = projectList.stream() - .filter(it -> it.getText().contains(projectName)) - .flatMap( - it -> it.findElements( - By.className("e2e-project-edit-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in project list")); - WebElementClick(driver, editButton); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/project/edit")); - - createProjectForm = new CreateProjectForm(); - WebElementDeleteAndInput(driver, createProjectForm.inputProjectName, newProjectName); - createProjectForm.buttonSubmit.click(); - - waitForListPageAfterSubmit(); - return this; - } - - @SneakyThrows - public ProjectsPage buildProject(String projectName) { - waitForPageLoading(); - - WebElement buildBtn = projectList.stream() - .filter(it -> it.getText().contains(projectName)) - .flatMap( - it -> it.findElements(By.className("e2e-project-build-btn")).stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No build button in project list")); - WebElementClick(driver, buildBtn); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buildConfirmButton)); - - buildConfirmButton.click(); - - return this; - } - - @SneakyThrows - public ProjectsPage deleteProject(String projectName) { - waitForPageLoading(); - WebElement deleteBtn = projectList.stream() - .filter(it -> it.getText().contains(projectName)) - .flatMap( - it -> it - .findElements(By.className("e2e-project-delete-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in project list")); - WebElementClick(driver, deleteBtn); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); - - deleteConfirmButton.click(); - PageFactory.initElements(driver, this); - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(d -> isProjectListPage(d.getCurrentUrl())); - PageFactory.initElements(driver, this); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateProject)); - createProjectForm = new CreateProjectForm(); - } - - private void waitForListPageAfterSubmit() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(d -> isProjectListPage(d.getCurrentUrl())); - PageFactory.initElements(driver, this); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateProject)); - createProjectForm = new CreateProjectForm(); - } - - private boolean isProjectListPage(String url) { - return url.contains("/resource/project") - || (url.contains("/project") && !url.contains("/project/add") && !url.contains("/project/edit")); - } - - @Getter - private class CreateProjectForm { - - CreateProjectForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(id = "form_item_name") - public WebElement inputProjectName; - - @FindBy(xpath = "//div[contains(@codefield, 'repository')]//div[contains(@class, 'ant-select-selector')]") - public WebElement selectCveDropdown; - - @FindBys({ - @FindBy(css = "[codefield=repository]"), - @FindBy(className = "ant-select-item-option-content") - }) - public List selectCve; - - @FindBy(name = "url") - public WebElement inputProjectUrl; - - @FindBy(xpath = "//div[contains(@codefield, 'refs')]//div[contains(@class, 'ant-select-selector')]") - public WebElement selectBranchDropdown; - - @FindBys({ - @FindBy(css = "[codefield=refs]"), - @FindBy(className = "ant-select-item-option-content") - }) - public List selectRefs; - - @FindBy(id = "form_item_buildArgs") - public WebElement inputBuildArgument; - - @FindBy(id = "form_item_description") - public WebElement inputDescription; - - @FindBy(id = "e2e-project-submit-btn") - public WebElement buttonSubmit; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ResourcePage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ResourcePage.java deleted file mode 100644 index c2d3a78be2..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/ResourcePage.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.resource; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; -import org.apache.streampark.e2e.pages.common.NavBarPage.NavBarItem; - -import lombok.Getter; -import lombok.SneakyThrows; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -@Getter -public final class ResourcePage extends NavBarPage implements NavBarItem { - - @FindBy(className = "menu-item-resource_variable") - public WebElement menuVariables; - - @FindBy(className = "menu-item-resource_project") - public WebElement menuProjects; - - @FindBy(className = "menu-item-resource_upload") - public WebElement menuUploads; - - public ResourcePage(RemoteWebDriver driver) { - super(driver); - } - - @SneakyThrows - public T goToTab(Class tab) { - if (tab == VariablesPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuVariables)); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - menuVariables.click(); - return tab.cast(new VariablesPage(driver)); - } - - if (tab == ProjectsPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuProjects)); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - menuProjects.click(); - return tab.cast(new ProjectsPage(driver)); - } - - if (tab == UploadsPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuUploads)); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - menuUploads.click(); - return tab.cast(new UploadsPage(driver)); - } - - throw new UnsupportedOperationException("Unknown tab: " + tab.getName()); - } - - public interface Tab { - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/UploadsPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/UploadsPage.java deleted file mode 100644 index eb8be84422..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/UploadsPage.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.resource; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; - -import lombok.Getter; -import lombok.SneakyThrows; -import org.openqa.selenium.By; -import org.openqa.selenium.Keys; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class UploadsPage extends NavBarPage implements ResourcePage.Tab { - - @FindBy(id = "e2e-upload-create-btn") - public WebElement buttonCreateResource; - - public final CreateUploadForm createUploadForm = new CreateUploadForm(); - - @FindBy(className = "ant-table-tbody") - public List resourceList; - - @FindBy(className = "swal2-html-container") - public List errorMessageList; - - @FindBy(xpath = "//button[contains(text(), 'OK')]") - public WebElement errorMessageConfirmButton; - - @FindBy(className = "e2e-upload-delete-confirm") - public WebElement deleteConfirmButton; - - public UploadsPage(RemoteWebDriver driver) { - super(driver); - } - - @SneakyThrows - public UploadsPage createUpload(String engineType, String resourceType, String resourceName, - String resource, - String description) { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateResource)); - buttonCreateResource.click(); - - // select engine type. - createUploadForm.btnSelectEngineTypeDropDown.click(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - createUploadForm.selectEngineType.stream() - .filter(e -> e.getText().equals(engineType)) - .findFirst() - .orElseThrow( - () -> new RuntimeException( - String.format("No %s in engineType dropdown list", engineType))) - .click(); - - // select resource type. - createUploadForm.btnSelectResourceTypeDropDown.click(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - createUploadForm.selectResourceType.stream() - .filter(e -> e.getText().equals(resourceType)) - .findFirst() - .orElseThrow( - () -> new RuntimeException( - String.format("No %s in resourceType dropdown list", resourceType))) - .click(); - - createUploadForm.inputResourceName.sendKeys(resourceName); - createUploadForm.textPom.sendKeys(resource); - createUploadForm.inputDescription.sendKeys(description); - - createUploadForm.buttonSubmit.click(); - - return this; - } - - @SneakyThrows - public UploadsPage editUpload(String engineType, String resourceType, String resourceName, - String resource, String description) { - waitForPageLoading(); - - resourceList.stream() - .filter(e -> e.getText().contains(resourceName)) - .flatMap( - it -> it.findElements(By.className("e2e-upload-edit-btn")).stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in resource list")) - .click(); - - // select engine type. - createUploadForm.btnSelectEngineTypeDropDown.click(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - createUploadForm.selectEngineType.stream() - .filter(e -> e.getText().equals(engineType)) - .findFirst() - .orElseThrow( - () -> new RuntimeException( - String.format("No %s in engineType dropdown list", resourceType))) - .click(); - - // select resource type. - createUploadForm.btnSelectResourceTypeDropDown.click(); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - createUploadForm.selectResourceType.stream() - .filter(e -> e.getText().equals(resourceType)) - .findFirst() - .orElseThrow( - () -> new RuntimeException( - String.format("No %s in resourceType dropdown list", resourceType))) - .click(); - - createUploadForm.textPom.sendKeys(Keys.chord(Keys.CONTROL, "a"), Keys.DELETE); - createUploadForm.textPom.sendKeys(resource); - createUploadForm.inputDescription.clear(); - createUploadForm.inputDescription.sendKeys(description); - - createUploadForm.buttonSubmit.click(); - - return this; - } - - public UploadsPage deleteUpload(String resourceName) { - waitForPageLoading(); - - resourceList.stream() - .filter(e -> e.getText().contains(resourceName)) - .flatMap( - it -> it.findElements(By.className("e2e-upload-delete-btn")).stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in resource list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); - - deleteConfirmButton.click(); - - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/resource/upload")); - } - - @Getter - public class CreateUploadForm { - - CreateUploadForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(xpath = "//*[@id='form_item_engineType']/ancestor::div[contains(@class, 'ant-select-selector')]") - private WebElement btnSelectEngineTypeDropDown; - - @FindBy(xpath = "//*[@id='form_item_engineType']//following::div[@class='ant-select-item-option-content']") - public List selectEngineType; - - @FindBy(xpath = "//*[@id='form_item_resourceType']/ancestor::div[contains(@class, 'ant-select-selector')]") - public WebElement btnSelectResourceTypeDropDown; - - @FindBy(xpath = "//*[@id='form_item_resourceType']//following::div[@class='ant-select-item-option-content']") - public List selectResourceType; - - @FindBy(id = "upload_form_resourceName") - public WebElement inputResourceName; - - @FindBy(css = "textarea.inputarea.monaco-mouse-cursor-text") - public WebElement textPom; - - @FindBy(id = "upload_form_description") - public WebElement inputDescription; - - @FindBy(className = "e2e-upload-submit-btn") - public WebElement buttonSubmit; - - @FindBy(className = "e2e-upload-cancel-btn") - public WebElement buttonCancel; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/VariablesPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/VariablesPage.java deleted file mode 100644 index 65e50d752b..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/resource/VariablesPage.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.resource; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; - -import lombok.Getter; -import org.openqa.selenium.By; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class VariablesPage extends NavBarPage implements ResourcePage.Tab { - - @FindBy(id = "e2e-var-create-btn") - public WebElement buttonCreateVariable; - - @FindBy(className = "ant-table-tbody") - public List variableList; - - @FindBy(className = "swal2-html-container") - public List errorMessageList; - - @FindBy(xpath = "//button[contains(@class, 'swal2-confirm')]") - public WebElement errorMessageConfirmButton; - - @FindBy(className = "e2e-var-delete-confirm") - public WebElement deleteConfirmButton; - - public final CreateVariableForm createVariableForm = new CreateVariableForm(); - - public VariablesPage(RemoteWebDriver driver) { - super(driver); - } - - public VariablesPage createVariable(String variableCode, String variableValue, String description, - boolean notVisible) { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateVariable)); - buttonCreateVariable.click(); - createVariableForm.inputVariableCode.sendKeys(variableCode); - createVariableForm.inputVariableValue.sendKeys(variableValue); - createVariableForm.inputDescription.sendKeys(description); - if (notVisible) { - createVariableForm.buttonDesensitization.click(); - } - - createVariableForm.buttonSubmit.click(); - - return this; - } - - public VariablesPage editVariable(String variableCode, String variableValue, String description, - boolean notVisible) { - waitForPageLoading(); - - variableList.stream() - .filter(it -> it.getText().contains(variableCode)) - .flatMap( - it -> it.findElements( - By.className("e2e-var-edit-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in variable list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(createVariableForm.buttonSubmit)); - createVariableForm.inputVariableValue.clear(); - createVariableForm.inputVariableValue.sendKeys(variableValue); - createVariableForm.inputDescription.clear(); - createVariableForm.inputDescription.sendKeys(description); - if (notVisible) { - createVariableForm.buttonDesensitization.click(); - } - createVariableForm.buttonSubmit.click(); - return this; - } - - public VariablesPage deleteVariable(String variableCode) { - waitForPageLoading(); - - variableList.stream() - .filter(it -> it.getText().contains(variableCode)) - .flatMap( - it -> it.findElements( - By.className("e2e-var-delete-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in variable list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); - - deleteConfirmButton.click(); - - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/resource/variable")); - } - - @Getter - public class CreateVariableForm { - - CreateVariableForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(id = "VariableForm_variableCode") - public WebElement inputVariableCode; - - @FindBy(id = "VariableForm_variableValue") - public WebElement inputVariableValue; - - @FindBy(id = "VariableForm_description") - public WebElement inputDescription; - - @FindBy(id = "VariableForm_desensitization") - public WebElement buttonDesensitization; - - @FindBy(className = "e2e-var-submit-btn") - public WebElement buttonSubmit; - - @FindBy(className = "e2e-var-cancel-btn") - public WebElement buttonCancel; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/ExternalLinkPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/ExternalLinkPage.java deleted file mode 100644 index 5046160dc9..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/ExternalLinkPage.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; - -import lombok.Getter; -import org.openqa.selenium.By; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class ExternalLinkPage extends NavBarPage implements SettingPage.Tab { - - @FindBy(xpath = "//span[contains(., 'External Link')]/..//button[contains(@class, 'ant-btn-dashed')]/span[contains(text(), 'Add New')]") - public WebElement buttonCreateExternalLink; - - @FindBy(xpath = "//tbody[contains(@class, 'ant-table-tbody')]") - public List externalLinkList; - - @FindBy(className = "swal2-html-container") - public List errorMessageList; - - @FindBy(xpath = "//button[contains(text(), 'OK')]") - public WebElement errorMessageConfirmButton; - - @FindBy(xpath = "//button[contains(@class, 'ant-btn')]/span[contains(., 'OK')]") - public WebElement deleteConfirmButton; - - public final CreateExternalLinkForm createExternalLinkForm = new CreateExternalLinkForm(); - - public ExternalLinkPage(RemoteWebDriver driver) { - super(driver); - } - - public ExternalLinkPage createExternalLink(String label, String name, String color, String link) { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateExternalLink)); - buttonCreateExternalLink.click(); - createExternalLinkForm.inputLabel.sendKeys(label); - createExternalLinkForm.inputName.sendKeys(name); - createExternalLinkForm.inputColor.sendKeys(color); - createExternalLinkForm.inputLink.sendKeys(link); - - createExternalLinkForm.buttonSubmit.click(); - - return this; - } - - public ExternalLinkPage editExternalLink(String label, String editLabel, String editName, String color, - String editLink) { - waitForPageLoading(); - - externalLinkList.stream() - .filter(it -> it.getText().contains(label)) - .flatMap( - it -> it.findElements(By.className("e2e-extlink-edit-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in external link list")) - .click(); - - createExternalLinkForm.inputLabel.clear(); - createExternalLinkForm.inputLabel.sendKeys(editLabel); - createExternalLinkForm.inputName.clear(); - createExternalLinkForm.inputName.sendKeys(editName); - createExternalLinkForm.inputColor.clear(); - createExternalLinkForm.inputColor.sendKeys(color); - createExternalLinkForm.inputLink.clear(); - createExternalLinkForm.inputLink.sendKeys(editLink); - - createExternalLinkForm.buttonSubmit.click(); - return this; - } - - public ExternalLinkPage deleteExternalLink(String label) { - waitForPageLoading(); - - externalLinkList.stream() - .filter(it -> it.getText().contains(label)) - .flatMap( - it -> it - .findElements(By.className("e2e-extlink-delete-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in External link list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); - - deleteConfirmButton.click(); - - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/setting/extlink")); - } - - @Getter - public class CreateExternalLinkForm { - - CreateExternalLinkForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(id = "form_item_badgeLabel") - public WebElement inputLabel; - - @FindBy(id = "form_item_badgeName") - public WebElement inputName; - - @FindBy(id = "form_item_badgeColor") - public WebElement inputColor; - - @FindBy(id = "form_item_linkUrl") - public WebElement inputLink; - - @FindBy(className = "e2e-extlink-submit-btn") - public WebElement buttonSubmit; - - @FindBy(className = "e2e-extlink-cancel-btn") - public WebElement buttonCancel; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/SettingPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/SettingPage.java deleted file mode 100644 index 24e68f2b7b..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/SettingPage.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; -import org.apache.streampark.e2e.pages.setting.alarm.AlarmPage; -import org.apache.streampark.e2e.pages.setting.env.EnvironmentPage; - -import lombok.Getter; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -@Getter -public class SettingPage extends NavBarPage implements NavBarPage.NavBarItem { - - @FindBy(xpath = "//span[contains(@class, 'streampark-simple-menu-sub-title') and contains(text(), 'Yarn Queue')]//..") - public WebElement menuYarnQueueManagement; - - @FindBy(xpath = "//span[contains(@class, 'streampark-simple-menu-sub-title') and contains(text(), 'External Link')]//..") - public WebElement menuExternalLinkManagement; - - @FindBy(xpath = "//span[contains(@class, 'streampark-simple-menu-sub-title') and contains(text(), 'Alarms')]//..") - public WebElement menuAlarmManagement; - - @FindBy(xpath = "//span[contains(@class, 'streampark-simple-menu-sub-title') and contains(text(), 'Environments')]//..") - public WebElement menuEnvManagement; - - public SettingPage(RemoteWebDriver driver) { - super(driver); - } - - public T goToTab(Class tab) { - if (tab == YarnQueuePage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuYarnQueueManagement)); - menuYarnQueueManagement.click(); - return tab.cast(new YarnQueuePage(driver)); - } - - if (tab == ExternalLinkPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuExternalLinkManagement)); - menuExternalLinkManagement.click(); - return tab.cast(new ExternalLinkPage(driver)); - } - - if (tab == AlarmPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuAlarmManagement)); - menuAlarmManagement.click(); - return tab.cast(new AlarmPage(driver)); - } - - if (tab == EnvironmentPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuEnvManagement)); - menuEnvManagement.click(); - return tab.cast(new EnvironmentPage(driver)); - } - - throw new UnsupportedOperationException("Unknown tab: " + tab.getName()); - } - - public interface Tab { - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/YarnQueuePage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/YarnQueuePage.java deleted file mode 100644 index 1a4abaaa39..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/YarnQueuePage.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; - -import lombok.Getter; -import org.openqa.selenium.By; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class YarnQueuePage extends NavBarPage implements SettingPage.Tab { - - @FindBy(id = "e2e-yarnqueue-create-btn") - public WebElement buttonCreateYarnQueue; - - @FindBy(xpath = "//tbody[contains(@class, 'ant-table-tbody')]") - public List yarnQueueList; - - @FindBy(className = "ant-form-item-explain-error") - public List errorMessageList; - - @FindBy(className = "e2e-yarnqueue-delete-confirm") - public WebElement deleteConfirmButton; - - public final CreateYarnQueueForm createYarnQueueForm = new CreateYarnQueueForm(); - - public YarnQueuePage(RemoteWebDriver driver) { - super(driver); - } - - public YarnQueuePage createYarnQueue(String queueLabel, String description) { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateYarnQueue)); - buttonCreateYarnQueue.click(); - - createYarnQueueForm.inputQueueLabel.sendKeys(queueLabel); - createYarnQueueForm.inputDescription.sendKeys(description); - - createYarnQueueForm.buttonOk.click(); - - return this; - } - - public YarnQueuePage editYarnQueue(String queueLabel, String editQueueLabel, String description) { - waitForPageLoading(); - - yarnQueueList.stream() - .filter(it -> it.getText().contains(queueLabel)) - .flatMap( - it -> it.findElements(By.className("e2e-yarnqueue-edit-btn")).stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in yarn queue list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(createYarnQueueForm.buttonOk)); - createYarnQueueForm.inputQueueLabel.clear(); - createYarnQueueForm.inputQueueLabel.sendKeys(editQueueLabel); - createYarnQueueForm.inputDescription.clear(); - createYarnQueueForm.inputDescription.sendKeys(description); - - createYarnQueueForm.buttonOk.click(); - - return this; - } - - public YarnQueuePage deleteYarnQueue(String queueLabel) { - waitForPageLoading(); - - yarnQueueList.stream() - .filter(it -> it.getText().contains(queueLabel)) - .flatMap( - it -> it.findElements(By.xpath(".//button[contains(@tooltip, 'Delete')]")).stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in yarn queue list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); - - deleteConfirmButton.click(); - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/setting/yarn-queue")); - } - - @Getter - public class CreateYarnQueueForm { - - CreateYarnQueueForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(id = "YarnQueueEditForm_queueLabel") - public WebElement inputQueueLabel; - - @FindBy(id = "YarnQueueEditForm_description") - public WebElement inputDescription; - - @FindBy(className = "e2e-yarnqueue-submit-btn") - public WebElement buttonOk; - - @FindBy(className = "e2e-yarnqueue-cancel-btn") - public WebElement buttonCancel; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/AlarmPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/AlarmPage.java deleted file mode 100644 index 014595970d..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/AlarmPage.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.alarm; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; -import org.apache.streampark.e2e.pages.setting.SettingPage; - -import lombok.Getter; -import org.openqa.selenium.By; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class AlarmPage extends NavBarPage implements SettingPage.Tab { - - @FindBy(xpath = "//span[contains(., 'Alarms Setting')]/..//button[contains(@class, 'ant-btn-dashed')]/span[contains(text(), 'Add New')]") - public WebElement buttonCreateAlarm; - - @FindBy(xpath = "//div[@class='ant-row']") - public List alarmList; - - @FindBy(className = "ant-form-item-explain-error") - public List errorMessageList; - - @FindBy(xpath = "//button[contains(text(), 'Submit')]") - public WebElement errorMessageConfirmButton; - - @FindBy(xpath = "//button[contains(@class, 'ant-btn')]/span[contains(., 'Yes')]") - public WebElement deleteConfirmButton; - - public AlarmPage(RemoteWebDriver driver) { - super(driver); - } - - public AlertTypeDetailForm createAlarm() { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateAlarm)); - buttonCreateAlarm.click(); - - return new AlertTypeDetailForm(driver); - } - - public AlertTypeDetailForm editAlarm(String alarmName) { - waitForPageLoading(); - - alarmList.stream() - // Filter out cards containing a specific alertName. - .filter(card -> { - WebElement titleElement = new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOf(card - .findElement(By.xpath(".//div[@class='ant-card-head']//div[@class='ant-card-head-title']")))); - return titleElement.getText().contains(alarmName); - }) - // Find the eligible cards and click the edit button. - .flatMap(card -> { - List editButtons = card.findElements(By.xpath( - ".//button[.//span[contains(@class, 'anticon') and contains(@class, 'anticon-edit')]]")); - - // Make sure the button is loaded. - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(editButtons)); - - return editButtons.stream(); - }) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button found for alarm: " + alarmName)) - .click(); - - return new AlertTypeDetailForm(driver); - } - - public AlarmPage deleteAlarm(String alarmName) { - waitForPageLoading(); - - alarmList.stream() - // Filter out cards containing a specific alertName. - .filter(card -> { - WebElement titleElement = new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOf(card - .findElement(By.xpath(".//div[@class='ant-card-head']//div[@class='ant-card-head-title']")))); - return titleElement.getText().contains(alarmName); - }) - // Find the eligible cards and click the delete button. - .flatMap(card -> { - List deleteButtons = card.findElements(By.xpath( - ".//button[.//span[contains(@class, 'anticon') and contains(@class, 'anticon-delete')]]")); - - // Make sure the button is loaded. - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(deleteButtons)); - - return deleteButtons.stream(); - }) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button found for alarm: " + alarmName)) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); - - deleteConfirmButton.click(); - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/setting/alarm")); - } - -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/AlertTypeDetailForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/AlertTypeDetailForm.java deleted file mode 100644 index 3db3fb588c..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/AlertTypeDetailForm.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.alarm; - -import org.apache.streampark.e2e.pages.common.Constants; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class AlertTypeDetailForm { - - public WebDriver driver; - - @FindBys({ - @FindBy(xpath = "//div[contains(@class, 'ant-select-selector')]"), - @FindBy(xpath = ".//input[@id='form_item_alertType']") - }) - public WebElement btnAlertTypeDropdown; - - @FindBy(xpath = "//*[@id='form_item_alertType']//following::div[@class='ant-select-item-option-content']") - public List selectAlertType; - - public AlertTypeDetailForm(WebDriver driver) { - PageFactory.initElements(driver, this); - this.driver = driver; - } - - @SuppressWarnings("unchecked") - public T addAlertType(AlertTypeEnum alertTypeEnum) { - btnAlertTypeDropdown.click(); - switch (alertTypeEnum) { - case EMAIL: - selectByAlertType(alertTypeEnum.desc); - return (T) new EmailAlertForm(this); - case DINGTALK: - selectByAlertType(alertTypeEnum.desc); - return (T) new DingTalkAlertForm(this); - case WECHAT: - selectByAlertType(alertTypeEnum.desc); - return (T) new WeChatAlertForm(this); - case SMS: - // ignore. - case LARK: - selectByAlertType(alertTypeEnum.desc); - return (T) new LarkAlertForm(this); - default: - throw new UnsupportedOperationException( - String.format("Unsupported alert type %s", alertTypeEnum.desc)); - } - } - - private void selectByAlertType(String alertType) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(selectAlertType)); - selectAlertType.stream() - .filter(e -> e.getText().equals(alertType)) - .findFirst() - .orElseThrow( - () -> new RuntimeException( - String.format("No %s in alertType dropdown list", alertType))) - .click(); - } - - @Getter - public enum AlertTypeEnum { - - EMAIL("E-mail"), - DINGTALK("Ding Talk"), - WECHAT("WeChat"), - LARK("Lark"), - SMS("SMS"); - public final String desc; - - AlertTypeEnum(String desc) { - this.desc = desc; - } - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/CommonForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/CommonForm.java deleted file mode 100644 index 6dfd4dff4e..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/CommonForm.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.alarm; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.PageFactory; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; - -@Getter -public abstract class CommonForm { - - public WebDriver driver; - - @FindBy(id = "form_item_alertName") - public WebElement inputAlertName; - - @FindBy(className = "e2e-alert-submit-btn") - public WebElement buttonSubmit; - - @FindBy(className = "e2e-alert-cancel-btn") - public WebElement buttonCancel; - - private final AlertTypeDetailForm parent; - - CommonForm(AlertTypeDetailForm alertTypeDetailForm) { - final WebDriver driver = alertTypeDetailForm.driver; - PageFactory.initElements(driver, this); - this.parent = alertTypeDetailForm; - } - - public CommonForm alertName(String alertName) { - WebElementDeleteAndInput(inputAlertName, alertName); - return this; - } - - public AlertTypeDetailForm submit() { - buttonSubmit.click(); - return parent; - } - - public AlertTypeDetailForm cancel() { - buttonCancel.click(); - return parent; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/DingTalkAlertForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/DingTalkAlertForm.java deleted file mode 100644 index 2e76593c74..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/DingTalkAlertForm.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.alarm; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; - -@Getter -public class DingTalkAlertForm extends CommonForm { - - public WebDriver driver; - - @FindBy(id = "form_item_alertDingURL") - public WebElement inputDingTalkURL; - - @FindBy(id = "form_item_dingtalkToken") - public WebElement inputDingTalkToken; - - @FindBy(id = "form_item_dingtalkSecretEnable") - public WebElement btnDingTalkSecret; - - @FindBy(id = "form_item_dingtalkSecretToken") - public WebElement inputDingTalkSecretToken; - - @FindBy(id = "form_item_alertDingUser") - public WebElement inputDingTalkReceiveUser; - - @FindBy(id = "form_item_dingtalkIsAtAll") - public WebElement btnDingTalkEffectToAllUsers; - - public DingTalkAlertForm(AlertTypeDetailForm alertTypeDetailForm) { - super(alertTypeDetailForm); - this.driver = alertTypeDetailForm.driver; - } - - public DingTalkAlertForm url(String url) { - WebElementDeleteAndInput(inputDingTalkURL, url); - return this; - } - - public DingTalkAlertForm token(String token) { - WebElementDeleteAndInput(inputDingTalkToken, token); - return this; - } - - public DingTalkAlertForm secretToken(String secretToken) { - WebElementDeleteAndInput(inputDingTalkSecretToken, secretToken); - return this; - } - - public DingTalkAlertForm receiveUser(String receiveUser) { - WebElementDeleteAndInput(inputDingTalkReceiveUser, receiveUser); - return this; - } - - public DingTalkAlertForm effectToAllUsers() { - btnDingTalkEffectToAllUsers.click(); - return this; - } - - public DingTalkAlertForm secretEnable() { - btnDingTalkSecret.click(); - return this; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/EmailAlertForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/EmailAlertForm.java deleted file mode 100644 index 703eccd7fe..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/EmailAlertForm.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.alarm; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; - -@Getter -public class EmailAlertForm extends CommonForm { - - public WebDriver driver; - - @FindBy(id = "form_item_alertEmail") - public WebElement inputEmail; - - public EmailAlertForm(AlertTypeDetailForm alertTypeDetailForm) { - super(alertTypeDetailForm); - this.driver = alertTypeDetailForm.driver; - } - - public EmailAlertForm email(String email) { - WebElementDeleteAndInput(inputEmail, email); - return this; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/LarkAlertForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/LarkAlertForm.java deleted file mode 100644 index c599c0600b..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/LarkAlertForm.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.alarm; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; - -@Getter -public class LarkAlertForm extends CommonForm { - - public WebDriver driver; - - @FindBy(id = "form_item_larkToken") - public WebElement inputLarkToken; - - @FindBy(id = "form_item_larkSecretToken") - public WebElement inputLarkSecretToken; - - @FindBy(id = "form_item_larkSecretEnable") - public WebElement btnLarkSecret; - - @FindBy(id = "form_item_larkIsAtAll") - public WebElement btnLarkEffectToAllUsers; - - public LarkAlertForm(AlertTypeDetailForm alertTypeDetailForm) { - super(alertTypeDetailForm); - this.driver = alertTypeDetailForm.driver; - } - - public LarkAlertForm token(String token) { - WebElementDeleteAndInput(inputLarkToken, token); - return this; - } - - public LarkAlertForm secretToken(String secretToken) { - WebElementDeleteAndInput(inputLarkSecretToken, secretToken); - return this; - } - - public LarkAlertForm secretEnable() { - btnLarkSecret.click(); - return this; - } - - public LarkAlertForm effectToAllUsers() { - btnLarkEffectToAllUsers.click(); - return this; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/SMSAlertForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/SMSAlertForm.java deleted file mode 100644 index 5e14eb5557..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/alarm/SMSAlertForm.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.alarm; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; - -@Getter -public class SMSAlertForm extends CommonForm { - - public WebDriver driver; - - public SMSAlertForm(AlertTypeDetailForm alertTypeDetailForm) { - super(alertTypeDetailForm); - this.driver = alertTypeDetailForm.driver; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/CommonForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/CommonForm.java deleted file mode 100644 index b787fcf738..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/CommonForm.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.env; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.support.PageFactory; - -@Getter -public abstract class CommonForm { - - public WebDriver driver; - - public final EnvironmentDetailForm parent; - - public CommonForm(EnvironmentDetailForm environmentDetailForm) { - final WebDriver driver = environmentDetailForm.driver; - PageFactory.initElements(driver, this); - this.parent = environmentDetailForm; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/DockerSettingForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/DockerSettingForm.java deleted file mode 100644 index 47c7aadce5..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/DockerSettingForm.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.env; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementClick; -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; - -@Getter -public class DockerSettingForm extends CommonForm { - - public WebDriver driver; - - @FindBy(id = "SettingForm_address") - public WebElement inputAddress; - - @FindBy(id = "SettingForm_namespace") - public WebElement inputNamespace; - - @FindBy(id = "SettingForm_username") - public WebElement inputUsername; - - @FindBy(xpath = "//input[@placeholder='Docker Password']") - public WebElement inputPassword; - - @FindBy(xpath = "//div[contains(@class, 'ant-modal-title') and contains(., 'Docker Setting')]/../..//button[contains(@class, 'ant-btn')]//span[contains(text(), 'OK')]") - public WebElement buttonOk; - - @FindBy(xpath = "//div[contains(@class, 'ant-modal-title') and contains(., 'Docker Setting')]/../..//button[contains(@class, 'ant-btn')]//span[contains(text(), 'Cancel')]") - public WebElement buttonCancel; - - DockerSettingForm(EnvironmentDetailForm environmentDetailForm) { - super(environmentDetailForm); - this.driver = environmentDetailForm.driver; - } - - public DockerSettingForm address(String address) { - WebElementDeleteAndInput(inputAddress, address); - return this; - } - - public DockerSettingForm namespace(String namespace) { - WebElementDeleteAndInput(inputNamespace, namespace); - return this; - } - - public DockerSettingForm user(String user) { - WebElementDeleteAndInput(inputUsername, user); - return this; - } - - public DockerSettingForm password(String password) { - WebElementDeleteAndInput(inputPassword, password); - return this; - } - - public DockerSettingForm ok() { - WebElementClick(driver, buttonOk); - return this; - } - - public DockerSettingForm cancel() { - WebElementClick(driver, buttonCancel); - return this; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EmailSettingForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EmailSettingForm.java deleted file mode 100644 index 094363c793..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EmailSettingForm.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.env; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementClick; -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; - -@Getter -public class EmailSettingForm extends CommonForm { - - public WebDriver driver; - - @FindBy(id = "SettingForm_host") - public WebElement inputSmtpHost; - - @FindBy(xpath = "//input[@placeholder='Smtp Port']") - public WebElement inputSmtpPort; - - @FindBy(id = "SettingForm_from") - public WebElement inputAddress; - - @FindBy(id = "SettingForm_userName") - public WebElement inputUsername; - - @FindBy(xpath = "//input[@placeholder='Email Password']") - public WebElement inputPassword; - - @FindBy(id = "SettingForm_ssl") - public WebElement btnEnableSSL; - - @FindBy(xpath = "//div[contains(@class, 'ant-modal-title') and contains(., 'Sender Email Setting')]/../..//button[contains(@class, 'ant-btn')]//span[contains(text(), 'OK')]") - public WebElement buttonOk; - - @FindBy(xpath = "//div[contains(@class, 'ant-modal-title') and contains(., 'Sender Email Setting')]/../..//button[contains(@class, 'ant-btn')]//span[contains(text(), 'Cancel')]") - public WebElement buttonCancel; - - EmailSettingForm(EnvironmentDetailForm environmentDetailForm) { - super(environmentDetailForm); - this.driver = environmentDetailForm.driver; - } - - public EmailSettingForm host(String host) { - WebElementDeleteAndInput(inputSmtpHost, host); - return this; - } - - public EmailSettingForm port(String port) { - WebElementDeleteAndInput(inputSmtpPort, port); - return this; - } - - public EmailSettingForm address(String address) { - WebElementDeleteAndInput(inputAddress, address); - return this; - } - public EmailSettingForm user(String user) { - WebElementDeleteAndInput(inputUsername, user); - return this; - } - - public EmailSettingForm password(String password) { - WebElementDeleteAndInput(inputPassword, password); - return this; - } - - public EmailSettingForm ssl() { - WebElementClick(driver, btnEnableSSL); - return this; - } - - public EmailSettingForm ok() { - WebElementClick(driver, buttonOk); - return this; - } - - public EmailSettingForm cancel() { - WebElementClick(driver, buttonCancel); - return this; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EnvironmentDetailForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EnvironmentDetailForm.java deleted file mode 100644 index 10c416db16..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EnvironmentDetailForm.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.env; - -import org.apache.streampark.e2e.pages.common.Constants; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.WebDriverWait; - -@Getter -public class EnvironmentDetailForm { - - public WebDriver driver; - - public EnvironmentDetailForm(WebDriver driver) { - PageFactory.initElements(driver, this); - this.driver = driver; - } - - @SuppressWarnings("unchecked") - public T addSetting(EnvSettingTypeEnum envSettingTypeEnum) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION); - switch (envSettingTypeEnum) { - case Maven: - return (T) new MavenSettingForm(this); - case Email: - return (T) new EmailSettingForm(this); - case Docker: - return (T) new DockerSettingForm(this); - case Ingress: - return (T) new IngressSettingForm(this); - default: - throw new UnsupportedOperationException( - String.format("Unsupported Environment setting type %s", envSettingTypeEnum.desc)); - } - } - - @Getter - public enum EnvSettingTypeEnum { - - Maven("Maven"), - Docker("Docker"), - Email("Email"), - Ingress("Ingress"); - public final String desc; - - EnvSettingTypeEnum(String desc) { - this.desc = desc; - } - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EnvironmentPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EnvironmentPage.java deleted file mode 100644 index 5c81fea15c..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/EnvironmentPage.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.env; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; -import org.apache.streampark.e2e.pages.setting.SettingPage; - -import lombok.Getter; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementClick; - -@Getter -public class EnvironmentPage extends NavBarPage implements SettingPage.Tab { - - private static final Logger log = LoggerFactory.getLogger(EnvironmentPage.class); - - public EnvironmentPage(RemoteWebDriver driver) { - super(driver); - } - - @FindBys({ - @FindBy(xpath = "//h4[text()='Docker Setting']"), - @FindBy(xpath = "./ancestor::li[@class='ant-list-item']//button[contains(@class, 'ant-btn') and contains(@class, 'ant-btn-primary') and contains(@class, 'ant-btn-circle')]") - }) - public WebElement btnCreateDockerSetting; - - @FindBys({ - @FindBy(xpath = "//h4[text()='Alert Mailbox Setting']"), - @FindBy(xpath = "./ancestor::li[@class='ant-list-item']//button[contains(@class, 'ant-btn') and contains(@class, 'ant-btn-primary') and contains(@class, 'ant-btn-circle')]") - }) - public WebElement btnCreateEmailSetting; - - @FindBy(xpath = "//div[contains(@class, 'system-setting')]") - public List settingList; - - @FindBy(className = "swal2-container") - public List errorMessageList; - - @FindBy(xpath = "//button[contains(@class, 'swal2-confirm') and contains(@class, 'swal2-styled') and text()='OK']") - public WebElement errorMessageConfirmButton; - - public EnvironmentDetailForm createEnvironment(EnvironmentDetailForm.EnvSettingTypeEnum envSettingTypeEnum) { - waitForPageLoading(); - switch (envSettingTypeEnum) { - case Docker: - WebElementClick(driver, btnCreateDockerSetting); - break; - case Email: - WebElementClick(driver, btnCreateEmailSetting); - break; - case Maven: - case Ingress: - break; - default: - throw new UnsupportedOperationException( - String.format("Unsupported environment type %s", envSettingTypeEnum.desc)); - } - return new EnvironmentDetailForm(driver); - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/setting/system")); - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/IngressSettingForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/IngressSettingForm.java deleted file mode 100644 index 13157ea833..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/IngressSettingForm.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.env; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementClick; -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; - -@Getter -public class IngressSettingForm extends CommonForm { - - public WebDriver driver; - - @FindBys({ - @FindBy(xpath = "//h4[text()='Ingress domain address Setting']"), - @FindBy(xpath = "./ancestor::li[@class='ant-list-item']//button[contains(@class, 'ant-btn') and contains(@class, 'ant-btn-primary') and contains(@class, 'ant-btn-circle')]") - }) - public WebElement btnDomainAddressSetting; - - @FindBy(className = "ingress_mode_default") - public WebElement inputDomainAddress; - - IngressSettingForm(EnvironmentDetailForm environmentDetailForm) { - super(environmentDetailForm); - this.driver = environmentDetailForm.driver; - } - - public IngressSettingForm domainAddress(String domainAddress) { - WebElementClick(driver, btnDomainAddressSetting); - WebElementDeleteAndInput(inputDomainAddress, domainAddress); - WebElementClick(driver, btnDomainAddressSetting); - return this; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/MavenSettingForm.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/MavenSettingForm.java deleted file mode 100644 index 36b0d74aa1..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/setting/env/MavenSettingForm.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.setting.env; - -import lombok.Getter; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; - -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementClick; -import static org.apache.streampark.e2e.pages.common.CommonFactory.WebElementDeleteAndInput; - -@Getter -public class MavenSettingForm extends CommonForm { - - public WebDriver driver; - @FindBys({ - @FindBy(xpath = "//h4[text()='Maven Settings File Path']"), - @FindBy(xpath = "./ancestor::li[@class='ant-list-item']//button[contains(@class, 'ant-btn-primary')]") - }) - public WebElement btnSettingsFilePath; - - @FindBys({ - @FindBy(xpath = "//h4[text()='Maven Central Repository']"), - @FindBy(xpath = "./ancestor::li[@class='ant-list-item']//button[contains(@class, 'ant-btn-primary')]") - }) - public WebElement btnCentralRepository; - - @FindBys({ - @FindBy(xpath = "//h4[text()='Maven Central Repository Auth User']"), - @FindBy(xpath = "./ancestor::li[@class='ant-list-item']//button[contains(@class, 'ant-btn-primary')]") - }) - // Maven central repository authentication username. - public WebElement btnAuthUser; - - @FindBys({ - @FindBy(xpath = "//h4[text()='Maven Central Repository Auth Password']"), - @FindBy(xpath = "./ancestor::li[@class='ant-list-item']//button[contains(@class, 'ant-btn-primary')]") - }) - // Maven central repository authentication password. - public WebElement btnAuthPassword; - - @FindBy(className = "streampark_maven_settings") - public WebElement inputSettingsFilePath; - - @FindBy(className = "streampark_maven_central_repository") - public WebElement inputCentralRepository; - - @FindBy(className = "streampark_maven_auth_user") - public WebElement inputAuthUser; - - @FindBy(className = "streampark_maven_auth_password") - public WebElement inputAuthPassword; - - MavenSettingForm(EnvironmentDetailForm environmentDetailForm) { - super(environmentDetailForm); - this.driver = environmentDetailForm.driver; - } - - public MavenSettingForm filePath(String filePath) { - WebElementClick(driver, btnSettingsFilePath); - WebElementDeleteAndInput(inputSettingsFilePath, filePath); - WebElementClick(driver, btnSettingsFilePath); - return this; - } - - public MavenSettingForm centralRepository(String repository) { - WebElementClick(driver, btnCentralRepository); - WebElementDeleteAndInput(inputCentralRepository, repository); - WebElementClick(driver, btnCentralRepository); - return this; - } - - public MavenSettingForm authUser(String user) { - WebElementClick(driver, btnAuthUser); - WebElementDeleteAndInput(inputAuthUser, user); - WebElementClick(driver, btnAuthUser); - return this; - } - - public MavenSettingForm authPassword(String password) { - WebElementClick(driver, btnAuthPassword); - WebElementDeleteAndInput(inputAuthPassword, password); - WebElementClick(driver, btnAuthPassword); - return this; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/MemberManagementPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/MemberManagementPage.java deleted file mode 100644 index f9f7e693e7..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/MemberManagementPage.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.system; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; - -import lombok.Getter; -import org.openqa.selenium.By; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class MemberManagementPage extends NavBarPage implements SystemPage.Tab { - - @FindBy(id = "e2e-member-create-btn") - public WebElement buttonCreateMember; - - @FindBy(className = "ant-table-tbody") - public List memberList; - - @FindBy(id = "swal2-html-container") - public WebElement errorMessage; - - @FindBy(className = "swal2-confirm") - public WebElement errorMessageConfirmButton; - - @FindBy(className = "e2e-member-delete-confirm") - public WebElement deleteConfirmButton; - - public final CreateMemberForm createMemberForm = new CreateMemberForm(); - - public MemberManagementPage(RemoteWebDriver driver) { - super(driver); - } - - public MemberManagementPage createMember(String userName, String role) { - - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateMember)); - - buttonCreateMember.click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(createMemberForm.btnSelectUserNameDropDown)); - - createMemberForm.btnSelectUserNameDropDown.click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(createMemberForm.selectUserName)); - - createMemberForm.selectUserName.stream() - .filter(e -> e.getText().equals(userName)) - .findFirst() - .orElseThrow( - () -> new RuntimeException( - String.format("No %s in username dropdown list", userName))) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(createMemberForm.btnSelectRoleDropDown)); - - createMemberForm.btnSelectRoleDropDown.click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(createMemberForm.selectRole)); - - createMemberForm.selectRole.stream() - .filter(e -> e.getText().equals(role)) - .findFirst() - .orElseThrow( - () -> new RuntimeException( - String.format("No %s in role dropdown list", role))) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(createMemberForm.buttonSubmit)); - - createMemberForm.buttonSubmit.click(); - return this; - } - - public MemberManagementPage editMember(String userName, String role) { - waitForPageLoading(); - - memberList.stream() - .filter(it -> it.getText().contains(userName)) - .flatMap( - it -> it.findElements(By.className("e2e-member-edit-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in member list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(createMemberForm.btnSelectRoleDropDown)); - createMemberForm.btnSelectRoleDropDown.click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(createMemberForm.selectRole)); - - createMemberForm.selectRole.stream() - .filter(e -> e.getText().equals(role)) - .findFirst() - .orElseThrow( - () -> new RuntimeException( - String.format("No %s in role dropdown list", role))) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(createMemberForm.buttonSubmit)); - - createMemberForm.buttonSubmit.click(); - return this; - } - - public MemberManagementPage deleteMember(String userName) { - waitForPageLoading(); - - memberList.stream() - .filter(it -> it.getText().contains(userName)) - .flatMap( - it -> it.findElements(By.className("e2e-member-delete-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in member list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); - - deleteConfirmButton.click(); - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/system/member")); - } - - @Getter - public class CreateMemberForm { - - CreateMemberForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(css = "div[optionfiltergroup='username'][codefield='userName']") - public WebElement btnSelectUserNameDropDown; - - @FindBys({ - @FindBy(css = "[codefield='userName']"), - @FindBy(className = "ant-select-item-option-content") - }) - public List selectUserName; - - @FindBy(css = "[codefield='roleId']") - public WebElement btnSelectRoleDropDown; - - @FindBys({ - @FindBy(css = "[codefield='roleId']"), - @FindBy(className = "ant-select-item-option-content") - }) - public List selectRole; - - @FindBy(className = "e2e-member-submit-btn") - public WebElement buttonSubmit; - - @FindBy(className = "e2e-member-cancel-btn") - public WebElement buttonCancel; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/RoleManagementPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/RoleManagementPage.java deleted file mode 100644 index 4c935aae02..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/RoleManagementPage.java +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.system; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; - -import lombok.Getter; -import org.openqa.selenium.By; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; -import java.util.Objects; - -@Getter -public class RoleManagementPage extends NavBarPage implements SystemPage.Tab { - - @FindBy(id = "e2e-role-create-btn") - public WebElement buttonCreateRole; - - @FindBy(className = "ant-table-tbody") - public List roleList; - - @FindBy(className = "ant-form-item-explain-error") - public List errorMessageList; - - @FindBy(className = "e2e-role-delete-confirm") - public WebElement deleteConfirmButton; - - public final CreateRoleForm createRoleForm = new CreateRoleForm(); - - public RoleManagementPage(RemoteWebDriver driver) { - super(driver); - } - - public RoleManagementPage createRole(String roleName, String description, - String menuName) { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateRole)); - - buttonCreateRole.click(); - - createRoleForm.inputRoleName.sendKeys(roleName); - createRoleForm.inputDescription.sendKeys(description); - editRoleMenu(menuName); - - createRoleForm.buttonSubmit.click(); - return this; - } - - public RoleManagementPage editRole(String roleName, String description, - String menuName) { - waitForPageLoading(); - - roleList.stream() - .filter(it -> it.getText().contains(roleName)) - .flatMap( - it -> it.findElements(By.className("e2e-role-edit-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in role list")) - .click(); - - createRoleForm.inputDescription.sendKeys(description); - editRoleMenu(menuName); - - createRoleForm.buttonSubmit.click(); - return this; - } - - public RoleManagementPage deleteRole(String roleName) { - - waitForPageLoading(); - - roleList.stream() - .filter(it -> it.getText().contains(roleName)) - .flatMap( - it -> it.findElements(By.className("e2e-role-delete-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in role list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); - - deleteConfirmButton.click(); - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/system/role")); - } - - private RoleManagementPage editRoleMenu(String menuName) { - createRoleForm.inputMenus.stream() - .filter(e -> Objects.equals( - e.findElement(By.xpath( - ".//span[contains(@class, 'streampark-tree__title') and contains(@class, 'pl-2')]")) - .getText(), - menuName)) - .findFirst() - .orElseThrow( - () -> new RuntimeException( - String.format("No %s in menus checkbox tree", menuName))) - .findElement(By.className("ant-tree-checkbox-inner")) - .click(); - return this; - } - - @Getter - public class CreateRoleForm { - - CreateRoleForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(id = "role_form_roleName") - public WebElement inputRoleName; - - @FindBy(id = "role_form_description") - public WebElement inputDescription; - - @FindBys({ - @FindBy(className = "ant-tree-list"), - @FindBy(className = "ant-tree-treenode") - }) - public List inputMenus; - - @FindBy(className = "e2e-role-pop-ok") - public WebElement buttonSubmit; - - @FindBy(className = "e2e-role-pop-cancel") - public WebElement buttonCancel; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/SystemPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/SystemPage.java deleted file mode 100644 index bbc43a7bd3..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/SystemPage.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.system; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; -import org.apache.streampark.e2e.pages.common.NavBarPage.NavBarItem; - -import lombok.Getter; -import lombok.SneakyThrows; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -@Getter -public final class SystemPage extends NavBarPage implements NavBarItem { - - @FindBy(className = "menu-item-system_user") - public WebElement menuUserManagement; - - @FindBy(className = "menu-item-system_token") - public WebElement menuTokenManagement; - - @FindBy(className = "menu-item-system_role") - public WebElement menuRoleManagement; - - @FindBy(className = "menu-item-system_team") - public WebElement menuTeamManagement; - - @FindBy(className = "menu-item-system_member") - public WebElement menuMemberManagement; - - public SystemPage(RemoteWebDriver driver) { - super(driver); - } - - @SneakyThrows - public T goToTab(Class tab) { - if (tab == UserManagementPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuUserManagement)); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - menuUserManagement.click(); - return tab.cast(new UserManagementPage(driver)); - } - - if (tab == TeamManagementPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuTeamManagement)); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - menuTeamManagement.click(); - return tab.cast(new TeamManagementPage(driver)); - } - - if (tab == RoleManagementPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuRoleManagement)); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - menuRoleManagement.click(); - return tab.cast(new RoleManagementPage(driver)); - } - if (tab == TokenManagementPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuTokenManagement)); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - menuTokenManagement.click(); - return tab.cast(new TokenManagementPage(driver)); - } - - if (tab == MemberManagementPage.class) { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(menuMemberManagement)); - Thread.sleep(Constants.DEFAULT_SLEEP_MILLISECONDS); - menuMemberManagement.click(); - return tab.cast(new MemberManagementPage(driver)); - } - - throw new UnsupportedOperationException("Unknown tab: " + tab.getName()); - } - - public interface Tab { - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/TeamManagementPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/TeamManagementPage.java deleted file mode 100644 index 5da30ea23f..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/TeamManagementPage.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.system; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; - -import lombok.Getter; -import org.openqa.selenium.By; -import org.openqa.selenium.Keys; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class TeamManagementPage extends NavBarPage implements SystemPage.Tab { - - @FindBy(id = "e2e-team-create-btn") - public WebElement buttonCreateTeam; - - @FindBy(className = "ant-table-tbody") - public List teamList; - - @FindBy(className = "swal2-html-container") - public List errorMessageList; - - @FindBy(xpath = "//button[contains(text(), 'OK')]") - public WebElement errorMessageConfirmButton; - - @FindBy(xpath = "//button[contains(@class, 'ant-btn')]/span[contains(., 'OK')]") - public WebElement deleteConfirmButton; - - public final CreateTeamForm createTeamForm = new CreateTeamForm(); - - public TeamManagementPage(RemoteWebDriver driver) { - super(driver); - } - - public void createTeam(String teamName, String description) { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateTeam)); - buttonCreateTeam.click(); - createTeamForm.inputTeamName.sendKeys(teamName); - createTeamForm.inputDescription.sendKeys(description); - - createTeamForm.buttonSubmit.click(); - } - - public TeamManagementPage editTeam(String teamName, String description) { - waitForPageLoading(); - - teamList.stream() - .filter(it -> it.getText().contains(teamName)) - .flatMap( - it -> it.findElements(By.className("e2e-team-edit-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in team list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(createTeamForm.buttonSubmit)); - createTeamForm.inputDescription.sendKeys(Keys.CONTROL + "a"); - createTeamForm.inputDescription.sendKeys(Keys.BACK_SPACE); - createTeamForm.inputDescription.sendKeys(description); - - createTeamForm.buttonSubmit.click(); - - return this; - } - - public TeamManagementPage deleteTeam(String teamName) { - waitForPageLoading(); - - teamList.stream() - .filter(it -> it.getText().contains(teamName)) - .flatMap( - it -> it.findElements(By.className("e2e-team-delete-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in team list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); - - deleteConfirmButton.click(); - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/system/team")); - } - - @Getter - public class CreateTeamForm { - - CreateTeamForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(id = "TeamEditForm_teamName") - public WebElement inputTeamName; - - @FindBy(id = "TeamEditForm_description") - public WebElement inputDescription; - - @FindBy(className = "e2e-team-submit-btn") - public WebElement buttonSubmit; - - @FindBy(className = "e2e-team-cancel-btn") - public WebElement buttonCancel; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/TokenManagementPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/TokenManagementPage.java deleted file mode 100644 index adab5fa936..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/TokenManagementPage.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.system; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; - -import lombok.Getter; -import org.openqa.selenium.By; -import org.openqa.selenium.Keys; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class TokenManagementPage extends NavBarPage implements SystemPage.Tab { - - @FindBy(id = "e2e-token-create-btn") - public WebElement buttonCreateToken; - - @FindBy(className = "ant-table-tbody") - public List tokenList; - - @FindBy(className = "e2e-token-delete-confirm") - public WebElement deleteConfirmButton; - - @FindBy(className = "ant-form-item-explain-error") - public WebElement errorMessageSearchLayout; - - public final CreateTokenForm createTokenForm = new CreateTokenForm(); - - public TokenManagementPage(RemoteWebDriver driver) { - super(driver); - } - - public TokenManagementPage createToken(String existUserName, String description) { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateToken)); - buttonCreateToken.click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(createTokenForm.inputUserName)); - createTokenForm.inputUserName.sendKeys(existUserName); - createTokenForm.inputUserName.sendKeys(Keys.RETURN); - - createTokenForm.inputDescription.sendKeys(description); - createTokenForm.buttonSubmit.click(); - return this; - } - - public TokenManagementPage copyToken(String existUserName) { - waitForPageLoading(); - - tokenList.stream() - .filter(it -> it.getText().contains(existUserName)) - .flatMap( - it -> it.findElements(By.className("e2e-token-copy-btn")).stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No Copy button in token list")) - .click(); - return this; - } - - public TokenManagementPage deleteToken(String existUserName) { - waitForPageLoading(); - - tokenList.stream() - .filter(it -> it.getText().contains(existUserName)) - .flatMap( - it -> it.findElements(By.className("e2e-token-delete-btn")).stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No delete button in token list")) - .click(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(deleteConfirmButton)); - deleteConfirmButton.click(); - - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/system/token")); - } - - @Getter - public class CreateTokenForm { - - CreateTokenForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(id = "form_item_userId") - public WebElement inputUserName; - - @FindBy(id = "form_item_description") - public WebElement inputDescription; - - @FindBy(className = "e2e-token-submit-btn") - public WebElement buttonSubmit; - - @FindBy(className = "e2e-token-cancel-btn") - public WebElement buttonCancel; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/UserManagementPage.java b/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/UserManagementPage.java deleted file mode 100644 index 9d20b57ebd..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/UserManagementPage.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.pages.system; - -import org.apache.streampark.e2e.pages.common.Constants; -import org.apache.streampark.e2e.pages.common.NavBarPage; -import org.apache.streampark.e2e.pages.system.entity.UserManagementStatus; -import org.apache.streampark.e2e.pages.system.entity.UserManagementUserType; - -import lombok.Getter; -import org.openqa.selenium.By; -import org.openqa.selenium.Keys; -import org.openqa.selenium.WebElement; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.openqa.selenium.support.FindBy; -import org.openqa.selenium.support.FindBys; -import org.openqa.selenium.support.PageFactory; -import org.openqa.selenium.support.ui.ExpectedConditions; -import org.openqa.selenium.support.ui.WebDriverWait; - -import java.util.List; - -@Getter -public class UserManagementPage extends NavBarPage implements SystemPage.Tab { - - @FindBy(id = "e2e-user-create-btn") - public WebElement buttonCreateUser; - - @FindBy(className = "ant-table-tbody") - public List userList; - - @FindBy(className = "ant-form-item-explain-error") - public List errorMessageList; - - public final CreateUserForm createUserForm = new CreateUserForm(); - - public UserManagementPage(RemoteWebDriver driver) { - super(driver); - } - - public UserManagementPage createUser( - String userName, - String nickName, - String password, - String email, - UserManagementUserType userManagementUserType) { - waitForPageLoading(); - - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.elementToBeClickable(buttonCreateUser)); - buttonCreateUser.click(); - createUserForm.inputUserName.sendKeys(userName); - createUserForm.inputNickName.sendKeys(nickName); - createUserForm.inputPassword.sendKeys(password); - createUserForm.inputEmail.sendKeys(email); - - createUserForm.btnSelectUserTypeDropdown.click(); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(createUserForm.selectUserType)); - createUserForm.selectUserType.stream() - .filter(e -> e.getText().equalsIgnoreCase(String.valueOf(userManagementUserType))) - .findFirst() - .orElseThrow( - () -> new RuntimeException( - String.format("No %s in userType dropdown list", - userManagementUserType))) - .click(); - - createUserForm.buttonSubmit.click(); - - return this; - } - - public UserManagementPage editUser( - String userName, - String email, - UserManagementUserType userManagementUserType, - UserManagementStatus userManagementStatus) { - waitForPageLoading(); - - userList - .stream() - .filter(it -> it.getText().contains(userName)) - .flatMap( - it -> it.findElements(By.className("e2e-user-edit-btn")) - .stream()) - .filter(WebElement::isDisplayed) - .findFirst() - .orElseThrow(() -> new RuntimeException("No edit button in user list")) - .click(); - - createUserForm.inputEmail.sendKeys(Keys.CONTROL + "a"); - createUserForm.inputEmail.sendKeys(Keys.BACK_SPACE); - createUserForm.inputEmail.sendKeys(email); - - createUserForm.btnSelectUserTypeDropdown.click(); - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.visibilityOfAllElements(createUserForm.selectUserType)); - createUserForm.selectUserType.stream() - .filter(e -> e.getText().equalsIgnoreCase(String.valueOf(userManagementUserType))) - .findFirst() - .orElseThrow( - () -> new RuntimeException( - String.format("No %s in userType dropdown list", - userManagementUserType))) - .click(); - - switch (userManagementStatus) { - case LOCKED: - createUserForm.radioLocked.click(); - break; - case EFFECTIVE: - createUserForm.radioEffective.click(); - break; - default: - throw new RuntimeException("Unknown user management status"); - } - - createUserForm.buttonSubmit.click(); - return this; - } - - private void waitForPageLoading() { - new WebDriverWait(driver, Constants.DEFAULT_WEBDRIVER_WAIT_DURATION) - .until(ExpectedConditions.urlContains("/system/user")); - } - - @Getter - public class CreateUserForm { - - CreateUserForm() { - PageFactory.initElements(driver, this); - } - - @FindBy(id = "formUserName") - public WebElement inputUserName; - - @FindBy(id = "form_item_nickName") - public WebElement inputNickName; - - @FindBy(id = "form_item_password") - public WebElement inputPassword; - - @FindBy(id = "form_item_email") - public WebElement inputEmail; - - @FindBys({ - @FindBy(css = "[codefield=userType]"), - @FindBy(className = "ant-select-item-option-content") - }) - public List selectUserType; - - @FindBy(css = "[codefield=userType] > .ant-select-selector") - public WebElement btnSelectUserTypeDropdown; - - @FindBy(xpath = "//label[contains(@class, 'ant-radio-wrapper')]/span[contains(., 'lock')]") - public WebElement radioLocked; - - @FindBy(xpath = "//label[contains(@class, 'ant-radio-wrapper')]/span[contains(., 'effective')]") - public WebElement radioEffective; - - @FindBy(className = "e2e-user-submit-btn") - public WebElement buttonSubmit; - - @FindBy(className = "e2e-user-cancel-btn") - public WebElement buttonCancel; - } -} diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/Dockerfile deleted file mode 100644 index 9443edc26d..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM apache/streampark:ci diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml deleted file mode 100644 index 35c74674a6..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/docker-compose.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -services: - streampark: - image: apache/streampark:ci-basic - command: bash bin/streampark.sh start_docker - build: - context: ./ - dockerfile: ./Dockerfile - ports: - - 10000:10000 - - 10030:10030 - environment: - - SPRING_PROFILES_ACTIVE=h2 - - TZ=Asia/Shanghai - privileged: true - restart: unless-stopped - networks: - - e2e - volumes: - - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ - - ${HOME}/.m2:/root/.m2 - - ./gitconfig:/root/.gitconfig:ro - healthcheck: - test: [ "CMD", "curl", "http://localhost:10000" ] - interval: 5s - timeout: 5s - retries: 120 - -networks: - e2e: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/gitconfig b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/gitconfig deleted file mode 100644 index 3e94fc6027..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/basic/gitconfig +++ /dev/null @@ -1,19 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -[http] - sslVerify = false diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/environment/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/environment/Dockerfile deleted file mode 100644 index 9443edc26d..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/environment/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM apache/streampark:ci diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/environment/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/environment/docker-compose.yaml deleted file mode 100644 index 934a0ffd18..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/environment/docker-compose.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -services: - postfix: - image: boky/postfix - container_name: postfix - hostname: testmail - domainname: email.com - environment: - - maildomain=email.com - - smtp_user=admin:streampark - - ALLOW_EMPTY_SENDER_DOMAINS=true - ports: - - "25:25" - privileged: true - restart: unless-stopped - networks: - - e2e - streampark: - image: apache/streampark:ci-basic - command: bash bin/streampark.sh start_docker - build: - context: ./ - dockerfile: ./Dockerfile - ports: - - 10000:10000 - - 10030:10030 - environment: - - SPRING_PROFILES_ACTIVE=h2 - - TZ=Asia/Shanghai - privileged: true - restart: unless-stopped - networks: - - e2e - volumes: - - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:10000" ] - interval: 5s - timeout: 5s - retries: 120 -networks: - e2e: - diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/Dockerfile deleted file mode 100644 index b1e093fd41..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM apache/streampark:ci diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/docker-compose.yaml deleted file mode 100644 index be496ff7c0..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-remote/docker-compose.yaml +++ /dev/null @@ -1,90 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -services: - jobmanager: - image: flink:1.16.3 - command: jobmanager - ports: - - "8081:8081" - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - networks: - - e2e - volumes: - - flink_data:/opt/flink - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:8081" ] - interval: 5s - timeout: 5s - retries: 120 - - taskmanager: - image: flink:1.16.3 - depends_on: - - jobmanager - command: taskmanager - scale: 1 - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - taskmanager.numberOfTaskSlots: 2 - networks: - - e2e - volumes: - - flink_data:/opt/flink - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:8081" ] - interval: 5s - timeout: 5s - retries: 120 - - streampark: - image: apache/streampark:ci - command: bash bin/streampark.sh start_docker - build: - context: ./ - dockerfile: ./Dockerfile - ports: - - 10000:10000 - - 10030:10030 - environment: - - SPRING_PROFILES_ACTIVE=h2 - - TZ=Asia/Shanghai - - FLINK_JOBMANAGER_URL=http://jobmanager:8081 - privileged: true - restart: unless-stopped - networks: - - e2e - volumes: - - flink_data:/opt/flink - - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:10000" ] - interval: 5s - timeout: 5s - retries: 120 -networks: - e2e: -volumes: - flink_data: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/Dockerfile deleted file mode 100644 index f123de65d6..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM apache/streampark:ci as base-image -FROM flink:1.16.3-scala_2.12-java8 as flink-image -FROM eclipse-temurin:11-jdk-jammy as jdk11-image - -# Install streampark -FROM sbloodys/hadoop:3.3.6 -COPY --from=base-image /streampark /streampark -RUN sudo chown -R hadoop.hadoop /streampark \ - && sed -i "s/hadoop-user-name: hdfs$/hadoop-user-name: hadoop/g" /streampark/conf/config.yaml - -# Install Flink -COPY --from=flink-image /opt/flink /flink-1.16.3 -RUN sudo chown -R hadoop.hadoop /flink-1.16.3 - -# Bundled JDK 11 from Eclipse Temurin for StreamPark Console (avoid CentOS 7 yum mirror retirement) -COPY --from=jdk11-image /opt/java/openjdk /usr/lib/jvm/jdk11 -ENV JAVA_HOME=/usr/lib/jvm/jdk11 -ENV PATH="${JAVA_HOME}/bin:${PATH}" diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.config b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.config deleted file mode 100644 index 441fa11187..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.config +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HADOOP_HOME=/opt/hadoop -CORE-SITE.XML_fs.default.name=hdfs://namenode -CORE-SITE.XML_fs.defaultFS=hdfs://namenode -HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 -HDFS-SITE.XML_dfs.replication=1 -MAPRED-SITE.XML_mapreduce.framework.name=yarn -MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager -YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false -YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 -YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false -YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle -YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage=99.9 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-am-resource-percent=0.9 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.yaml deleted file mode 100644 index 0412171f46..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.16-on-yarn/docker-compose.yaml +++ /dev/null @@ -1,163 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -services: - namenode: - image: apache/streampark-flink-1.16.3-on-yarn:ci - hostname: namenode - command: [ "hdfs", "namenode" ] - networks: - - e2e - build: - context: ./ - ports: - - "19870:9870" - env_file: - - docker-compose.config - environment: - ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://namenode:9870" ] - interval: 5s - timeout: 5s - retries: 120 - datanode: - image: apache/streampark-flink-1.16.3-on-yarn:ci - hostname: datanode - command: [ "hdfs", "datanode" ] - networks: - - e2e - build: - context: ./ - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://datanode:9864" ] - interval: 5s - timeout: 5s - retries: 120 - depends_on: - namenode: - condition: service_healthy - resourcemanager: - image: apache/streampark-flink-1.16.3-on-yarn:ci - hostname: resourcemanager - command: [ "yarn", "resourcemanager" ] - networks: - - e2e - build: - context: ./ - ports: - - "18088:8088" - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://resourcemanager:8088" ] - interval: 5s - timeout: 5s - retries: 120 - nodemanager: - image: apache/streampark-flink-1.16.3-on-yarn:ci - hostname: nodemanager - command: [ "yarn", "nodemanager" ] - networks: - - e2e - build: - context: ./ - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - depends_on: - resourcemanager: - condition: service_healthy - healthcheck: - test: [ "CMD", "curl", "http://nodemanager:8042" ] - interval: 5s - timeout: 5s - retries: 120 - streampark: - image: apache/streampark-flink-1.16.3-on-yarn:ci - hostname: streampark - command: - - sh - - -c - - /streampark/bin/streampark.sh start_docker - networks: - - e2e - build: - context: ./ - ports: - - "20000:10000" - environment: - - SPRING_PROFILES_ACTIVE=h2 - - TZ=Asia/Shanghai - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - depends_on: - namenode: - condition: service_healthy - datanode: - condition: service_healthy - resourcemanager: - condition: service_healthy - nodemanager: - condition: service_healthy - healthcheck: - test: [ "CMD", "curl", "http://streampark:10000" ] - interval: 5s - timeout: 5s - retries: 120 - -networks: - e2e: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-remote/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-remote/Dockerfile deleted file mode 100644 index b1e093fd41..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-remote/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM apache/streampark:ci diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-remote/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-remote/docker-compose.yaml deleted file mode 100644 index 724172e32d..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-remote/docker-compose.yaml +++ /dev/null @@ -1,90 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -services: - jobmanager: - image: flink:1.17.2 - command: jobmanager - ports: - - "8081:8081" - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - networks: - - e2e - volumes: - - flink_data:/opt/flink - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:8081" ] - interval: 5s - timeout: 5s - retries: 120 - - taskmanager: - image: flink:1.17.2 - depends_on: - - jobmanager - command: taskmanager - scale: 1 - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - taskmanager.numberOfTaskSlots: 2 - networks: - - e2e - volumes: - - flink_data:/opt/flink - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:8081" ] - interval: 5s - timeout: 5s - retries: 120 - - streampark: - image: apache/streampark:ci - command: bash bin/streampark.sh start_docker - build: - context: ./ - dockerfile: ./Dockerfile - ports: - - 10000:10000 - - 10030:10030 - environment: - - SPRING_PROFILES_ACTIVE=h2 - - TZ=Asia/Shanghai - - FLINK_JOBMANAGER_URL=http://jobmanager:8081 - privileged: true - restart: unless-stopped - networks: - - e2e - volumes: - - flink_data:/opt/flink - - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:10000" ] - interval: 5s - timeout: 5s - retries: 120 -networks: - e2e: -volumes: - flink_data: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/Dockerfile deleted file mode 100644 index 9fa3defd34..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM apache/streampark:ci as base-image -FROM flink:1.17.2-scala_2.12-java8 as flink-image -FROM eclipse-temurin:11-jdk-jammy as jdk11-image - -# Install streampark -FROM sbloodys/hadoop:3.3.6 -COPY --from=base-image /streampark /streampark -RUN sudo chown -R hadoop.hadoop /streampark \ - && sed -i "s/hadoop-user-name: hdfs$/hadoop-user-name: hadoop/g" /streampark/conf/config.yaml - -# Install Flink -COPY --from=flink-image /opt/flink /flink-1.17.2 -RUN sudo chown -R hadoop.hadoop /flink-1.17.2 - -# Bundled JDK 11 from Eclipse Temurin for StreamPark Console (avoid CentOS 7 yum mirror retirement) -COPY --from=jdk11-image /opt/java/openjdk /usr/lib/jvm/jdk11 -ENV JAVA_HOME=/usr/lib/jvm/jdk11 -ENV PATH="${JAVA_HOME}/bin:${PATH}" diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/docker-compose.config b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/docker-compose.config deleted file mode 100644 index 441fa11187..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/docker-compose.config +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HADOOP_HOME=/opt/hadoop -CORE-SITE.XML_fs.default.name=hdfs://namenode -CORE-SITE.XML_fs.defaultFS=hdfs://namenode -HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 -HDFS-SITE.XML_dfs.replication=1 -MAPRED-SITE.XML_mapreduce.framework.name=yarn -MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager -YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false -YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 -YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false -YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle -YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage=99.9 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-am-resource-percent=0.9 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/docker-compose.yaml deleted file mode 100644 index 3f4a68b71c..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.17-on-yarn/docker-compose.yaml +++ /dev/null @@ -1,163 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -services: - namenode: - image: apache/streampark-flink-1.17.2-on-yarn:ci - hostname: namenode - command: [ "hdfs", "namenode" ] - networks: - - e2e - build: - context: ./ - ports: - - "19870:9870" - env_file: - - docker-compose.config - environment: - ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://namenode:9870" ] - interval: 5s - timeout: 5s - retries: 120 - datanode: - image: apache/streampark-flink-1.17.2-on-yarn:ci - hostname: datanode - command: [ "hdfs", "datanode" ] - networks: - - e2e - build: - context: ./ - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://datanode:9864" ] - interval: 5s - timeout: 5s - retries: 120 - depends_on: - namenode: - condition: service_healthy - resourcemanager: - image: apache/streampark-flink-1.17.2-on-yarn:ci - hostname: resourcemanager - command: [ "yarn", "resourcemanager" ] - networks: - - e2e - build: - context: ./ - ports: - - "18088:8088" - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://resourcemanager:8088" ] - interval: 5s - timeout: 5s - retries: 120 - nodemanager: - image: apache/streampark-flink-1.17.2-on-yarn:ci - hostname: nodemanager - command: [ "yarn", "nodemanager" ] - networks: - - e2e - build: - context: ./ - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - depends_on: - resourcemanager: - condition: service_healthy - healthcheck: - test: [ "CMD", "curl", "http://nodemanager:8042" ] - interval: 5s - timeout: 5s - retries: 120 - streampark: - image: apache/streampark-flink-1.17.2-on-yarn:ci - hostname: streampark - command: - - sh - - -c - - /streampark/bin/streampark.sh start_docker - networks: - - e2e - build: - context: ./ - ports: - - "20000:10000" - environment: - - SPRING_PROFILES_ACTIVE=h2 - - TZ=Asia/Shanghai - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - depends_on: - namenode: - condition: service_healthy - datanode: - condition: service_healthy - resourcemanager: - condition: service_healthy - nodemanager: - condition: service_healthy - healthcheck: - test: [ "CMD", "curl", "http://streampark:10000" ] - interval: 5s - timeout: 5s - retries: 120 - -networks: - e2e: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-remote/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-remote/Dockerfile deleted file mode 100644 index b1e093fd41..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-remote/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM apache/streampark:ci diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-remote/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-remote/docker-compose.yaml deleted file mode 100644 index f44d1af386..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-remote/docker-compose.yaml +++ /dev/null @@ -1,90 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -services: - jobmanager: - image: flink:1.18.1 - command: jobmanager - ports: - - "8081:8081" - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - networks: - - e2e - volumes: - - flink_data:/opt/flink - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:8081" ] - interval: 5s - timeout: 5s - retries: 120 - - taskmanager: - image: flink:1.18.1 - depends_on: - - jobmanager - command: taskmanager - scale: 1 - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - taskmanager.numberOfTaskSlots: 2 - networks: - - e2e - volumes: - - flink_data:/opt/flink - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:8081" ] - interval: 5s - timeout: 5s - retries: 120 - - streampark: - image: apache/streampark:ci - command: bash bin/streampark.sh start_docker - build: - context: ./ - dockerfile: ./Dockerfile - ports: - - 10000:10000 - - 10030:10030 - environment: - - SPRING_PROFILES_ACTIVE=h2 - - TZ=Asia/Shanghai - - FLINK_JOBMANAGER_URL=http://jobmanager:8081 - privileged: true - restart: unless-stopped - networks: - - e2e - volumes: - - flink_data:/opt/flink - - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:10000" ] - interval: 5s - timeout: 5s - retries: 120 -networks: - e2e: -volumes: - flink_data: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/Dockerfile deleted file mode 100644 index 1569de40a3..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM apache/streampark:ci as base-image -FROM flink:1.18.1-scala_2.12-java8 as flink-image -FROM eclipse-temurin:11-jdk-jammy as jdk11-image - -# Install streampark -FROM sbloodys/hadoop:3.3.6 -COPY --from=base-image /streampark /streampark -RUN sudo chown -R hadoop.hadoop /streampark \ - && sed -i "s/hadoop-user-name: hdfs$/hadoop-user-name: hadoop/g" /streampark/conf/config.yaml - -# Install Flink -COPY --from=flink-image /opt/flink /flink-1.18.1 -RUN sudo chown -R hadoop.hadoop /flink-1.18.1 - -# Bundled JDK 11 from Eclipse Temurin for StreamPark Console (avoid CentOS 7 yum mirror retirement) -COPY --from=jdk11-image /opt/java/openjdk /usr/lib/jvm/jdk11 -ENV JAVA_HOME=/usr/lib/jvm/jdk11 -ENV PATH="${JAVA_HOME}/bin:${PATH}" diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/docker-compose.config b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/docker-compose.config deleted file mode 100644 index 441fa11187..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/docker-compose.config +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HADOOP_HOME=/opt/hadoop -CORE-SITE.XML_fs.default.name=hdfs://namenode -CORE-SITE.XML_fs.defaultFS=hdfs://namenode -HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 -HDFS-SITE.XML_dfs.replication=1 -MAPRED-SITE.XML_mapreduce.framework.name=yarn -MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager -YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false -YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 -YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false -YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle -YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage=99.9 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-am-resource-percent=0.9 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/docker-compose.yaml deleted file mode 100644 index a867ac1723..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.18-on-yarn/docker-compose.yaml +++ /dev/null @@ -1,163 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -services: - namenode: - image: apache/streampark-flink-1.18.1-on-yarn:ci - hostname: namenode - command: [ "hdfs", "namenode" ] - networks: - - e2e - build: - context: ./ - ports: - - "19870:9870" - env_file: - - docker-compose.config - environment: - ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://namenode:9870" ] - interval: 5s - timeout: 5s - retries: 120 - datanode: - image: apache/streampark-flink-1.18.1-on-yarn:ci - hostname: datanode - command: [ "hdfs", "datanode" ] - networks: - - e2e - build: - context: ./ - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://datanode:9864" ] - interval: 5s - timeout: 5s - retries: 120 - depends_on: - namenode: - condition: service_healthy - resourcemanager: - image: apache/streampark-flink-1.18.1-on-yarn:ci - hostname: resourcemanager - command: [ "yarn", "resourcemanager" ] - networks: - - e2e - build: - context: ./ - ports: - - "18088:8088" - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://resourcemanager:8088" ] - interval: 5s - timeout: 5s - retries: 120 - nodemanager: - image: apache/streampark-flink-1.18.1-on-yarn:ci - hostname: nodemanager - command: [ "yarn", "nodemanager" ] - networks: - - e2e - build: - context: ./ - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - depends_on: - resourcemanager: - condition: service_healthy - healthcheck: - test: [ "CMD", "curl", "http://nodemanager:8042" ] - interval: 5s - timeout: 5s - retries: 120 - streampark: - image: apache/streampark-flink-1.18.1-on-yarn:ci - hostname: streampark - command: - - sh - - -c - - /streampark/bin/streampark.sh start_docker - networks: - - e2e - build: - context: ./ - ports: - - "20000:10000" - environment: - - SPRING_PROFILES_ACTIVE=h2 - - TZ=Asia/Shanghai - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - depends_on: - namenode: - condition: service_healthy - datanode: - condition: service_healthy - resourcemanager: - condition: service_healthy - nodemanager: - condition: service_healthy - healthcheck: - test: [ "CMD", "curl", "http://streampark:10000" ] - interval: 5s - timeout: 5s - retries: 120 - -networks: - e2e: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-remote/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-remote/Dockerfile deleted file mode 100644 index b1e093fd41..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-remote/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM apache/streampark:ci diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-remote/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-remote/docker-compose.yaml deleted file mode 100644 index b16486e6b5..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-remote/docker-compose.yaml +++ /dev/null @@ -1,90 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -services: - jobmanager: - image: flink:1.20.1 - command: jobmanager - ports: - - "8081:8081" - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - networks: - - e2e - volumes: - - flink_data:/opt/flink - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:8081" ] - interval: 5s - timeout: 5s - retries: 120 - - taskmanager: - image: flink:1.20.1 - depends_on: - - jobmanager - command: taskmanager - scale: 1 - environment: - - | - FLINK_PROPERTIES= - jobmanager.rpc.address: jobmanager - taskmanager.numberOfTaskSlots: 2 - networks: - - e2e - volumes: - - flink_data:/opt/flink - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:8081" ] - interval: 5s - timeout: 5s - retries: 120 - - streampark: - image: apache/streampark:ci - command: bash bin/streampark.sh start_docker - build: - context: ./ - dockerfile: ./Dockerfile - ports: - - 10000:10000 - - 10030:10030 - environment: - - SPRING_PROFILES_ACTIVE=h2 - - TZ=Asia/Shanghai - - FLINK_JOBMANAGER_URL=http://jobmanager:8081 - privileged: true - restart: unless-stopped - networks: - - e2e - volumes: - - flink_data:/opt/flink - - ${HOME}/streampark_build_logs:/tmp/streampark/logs/build_logs/ - - /var/run/docker.sock:/var/run/docker.sock - healthcheck: - test: [ "CMD", "curl", "http://localhost:10000" ] - interval: 5s - timeout: 5s - retries: 120 -networks: - e2e: -volumes: - flink_data: diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/Dockerfile b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/Dockerfile deleted file mode 100644 index bcd5c1cc97..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM apache/streampark:ci as base-image -FROM flink:1.20.1-scala_2.12-java8 as flink-image -FROM eclipse-temurin:11-jdk-jammy as jdk11-image - -# Install streampark -FROM sbloodys/hadoop:3.3.6 -COPY --from=base-image /streampark /streampark -RUN sudo chown -R hadoop.hadoop /streampark \ - && sed -i "s/hadoop-user-name: hdfs$/hadoop-user-name: hadoop/g" /streampark/conf/config.yaml - -# Install Flink -COPY --from=flink-image /opt/flink /flink-1.20.1 -RUN sudo chown -R hadoop.hadoop /flink-1.20.1 - -# Bundled JDK 11 from Eclipse Temurin for StreamPark Console (avoid CentOS 7 yum mirror retirement) -COPY --from=jdk11-image /opt/java/openjdk /usr/lib/jvm/jdk11 -ENV JAVA_HOME=/usr/lib/jvm/jdk11 -ENV PATH="${JAVA_HOME}/bin:${PATH}" diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/docker-compose.config b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/docker-compose.config deleted file mode 100644 index 441fa11187..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/docker-compose.config +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -HADOOP_HOME=/opt/hadoop -CORE-SITE.XML_fs.default.name=hdfs://namenode -CORE-SITE.XML_fs.defaultFS=hdfs://namenode -HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 -HDFS-SITE.XML_dfs.replication=1 -MAPRED-SITE.XML_mapreduce.framework.name=yarn -MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME -YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager -YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false -YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 -YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false -YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle -YARN-SITE.XML_yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage=99.9 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-am-resource-percent=0.9 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= -CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false diff --git a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/docker-compose.yaml b/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/docker-compose.yaml deleted file mode 100644 index 73af8308be..0000000000 --- a/streampark-e2e/streampark-e2e-case/src/test/resources/docker/flink-1.20-on-yarn/docker-compose.yaml +++ /dev/null @@ -1,163 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -services: - namenode: - image: apache/streampark-flink-1.20.1-on-yarn:ci - hostname: namenode - command: [ "hdfs", "namenode" ] - networks: - - e2e - build: - context: ./ - ports: - - "19870:9870" - env_file: - - docker-compose.config - environment: - ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://namenode:9870" ] - interval: 5s - timeout: 5s - retries: 120 - datanode: - image: apache/streampark-flink-1.20.1-on-yarn:ci - hostname: datanode - command: [ "hdfs", "datanode" ] - networks: - - e2e - build: - context: ./ - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://datanode:9864" ] - interval: 5s - timeout: 5s - retries: 120 - depends_on: - namenode: - condition: service_healthy - resourcemanager: - image: apache/streampark-flink-1.20.1-on-yarn:ci - hostname: resourcemanager - command: [ "yarn", "resourcemanager" ] - networks: - - e2e - build: - context: ./ - ports: - - "18088:8088" - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - healthcheck: - test: [ "CMD", "curl", "http://resourcemanager:8088" ] - interval: 5s - timeout: 5s - retries: 120 - nodemanager: - image: apache/streampark-flink-1.20.1-on-yarn:ci - hostname: nodemanager - command: [ "yarn", "nodemanager" ] - networks: - - e2e - build: - context: ./ - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - depends_on: - resourcemanager: - condition: service_healthy - healthcheck: - test: [ "CMD", "curl", "http://nodemanager:8042" ] - interval: 5s - timeout: 5s - retries: 120 - streampark: - image: apache/streampark-flink-1.20.1-on-yarn:ci - hostname: streampark - command: - - sh - - -c - - /streampark/bin/streampark.sh start_docker - networks: - - e2e - build: - context: ./ - ports: - - "20000:10000" - environment: - - SPRING_PROFILES_ACTIVE=h2 - - TZ=Asia/Shanghai - env_file: - - docker-compose.config - logging: - driver: "json-file" - options: - max-size: "200m" - max-file: "1" - tty: true - stdin_open: true - restart: always - depends_on: - namenode: - condition: service_healthy - datanode: - condition: service_healthy - resourcemanager: - condition: service_healthy - nodemanager: - condition: service_healthy - healthcheck: - test: [ "CMD", "curl", "http://streampark:10000" ] - interval: 5s - timeout: 5s - retries: 120 - -networks: - e2e: diff --git a/streampark-e2e/streampark-e2e-core/pom.xml b/streampark-e2e/streampark-e2e-core/pom.xml deleted file mode 100644 index 606ceeab6c..0000000000 --- a/streampark-e2e/streampark-e2e-core/pom.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-e2e - 1.0-SNAPSHOT - - - streampark-e2e-core - diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/Constants.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/Constants.java deleted file mode 100644 index 3d57754c2d..0000000000 --- a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/Constants.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.core; - -import lombok.experimental.UtilityClass; - -import java.nio.file.Path; -import java.nio.file.Paths; - -@UtilityClass -public final class Constants { - - /** tmp directory path */ - public static final Path HOST_TMP_PATH = Paths.get(System.getProperty("java.io.tmpdir")); - - /** chrome download path in host */ - public static final Path HOST_CHROME_DOWNLOAD_PATH = HOST_TMP_PATH.resolve("download"); - - /** chrome download path in selenium/standalone-chrome-debug container */ - public static final String SELENIUM_CONTAINER_CHROME_DOWNLOAD_PATH = "/home/seluser/Downloads"; - -} diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamPark.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamPark.java deleted file mode 100644 index 50404d30a9..0000000000 --- a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamPark.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.core; - -import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; -import org.junit.jupiter.api.TestMethodOrder; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.junit.jupiter.Testcontainers; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Inherited; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Inherited -@Testcontainers -@Target(ElementType.TYPE) -@Retention(RetentionPolicy.RUNTIME) -@TestMethodOrder(OrderAnnotation.class) -@ExtendWith(StreamParkExtension.class) -public @interface StreamPark { - - String[] composeFiles(); -} diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java deleted file mode 100644 index 733256d771..0000000000 --- a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/StreamParkExtension.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.core; - -import com.google.common.base.Strings; -import com.google.common.net.HostAndPort; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.extension.AfterAllCallback; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.BeforeEachCallback; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.openqa.selenium.WebDriver; -import org.openqa.selenium.chrome.ChromeDriver; -import org.openqa.selenium.chrome.ChromeOptions; -import org.openqa.selenium.remote.RemoteWebDriver; -import org.testcontainers.Testcontainers; -import org.testcontainers.containers.BrowserWebDriverContainer; -import org.testcontainers.containers.ComposeContainer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.shaded.org.awaitility.Awaitility; -import org.testcontainers.utility.DockerImageName; - -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Modifier; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.testcontainers.containers.BrowserWebDriverContainer.VncRecordingMode.RECORD_ALL; -import static org.testcontainers.containers.VncRecordingContainer.VncRecordingFormat.MP4; - -@Slf4j -final class StreamParkExtension implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback { - - private final boolean LOCAL_MODE = Objects.equals(System.getProperty("local"), "true"); - - private final boolean M1_CHIP_FLAG = Objects.equals(System.getProperty("m1_chip"), "true"); - - private final boolean LOCAL_BROWSER = Objects.equals(System.getProperty("local_browser"), "true"); - - private final int LOCAL_PORT = 10001; - - private final int DOCKER_PORT = 10000; - - private RemoteWebDriver driver; - private ComposeContainer compose; - private BrowserWebDriverContainer browser; - private HostAndPort address; - private String rootPath; - - private Path record; - - private final String serviceName = "streampark"; - - @Override - @SuppressWarnings("UnstableApiUsage") - public void beforeAll(ExtensionContext context) throws IOException { - Awaitility.setDefaultTimeout(Duration.ofSeconds(120)); - Awaitility.setDefaultPollInterval(Duration.ofSeconds(2)); - - setRecordPath(); - - if (LOCAL_MODE) { - runInLocal(); - } else { - runInDockerContainer(context); - } - - if (LOCAL_BROWSER) { - driver = new ChromeDriver(new ChromeOptions()); - } else { - setBrowserContainerByOsName(); - - if (compose != null) { - Testcontainers.exposeHostPorts(compose.getServicePort(serviceName, DOCKER_PORT)); - browser.withAccessToHost(true); - } - browser.start(); - - driver = new RemoteWebDriver(browser.getSeleniumAddress(), new ChromeOptions()); - } - - driver - .manage() - .timeouts() - .implicitlyWait(Duration.ofSeconds(2)) - .pageLoadTimeout(Duration.ofSeconds(5)); - driver.manage().window().maximize(); - - driver.get(new URL("http", address.getHost(), address.getPort(), rootPath).toString()); - - if (!LOCAL_BROWSER) { - browser.beforeTest(new TestDescription(context)); - } - - final Class clazz = context.getRequiredTestClass(); - Stream.of(clazz.getDeclaredFields()) - .filter(it -> Modifier.isStatic(it.getModifiers())) - .filter(f -> WebDriver.class.isAssignableFrom(f.getType())) - .forEach(it -> setDriver(clazz, it)); - } - - private void runInLocal() { - Testcontainers.exposeHostPorts(LOCAL_PORT); - address = HostAndPort.fromParts(browserHost(), LOCAL_PORT); - rootPath = "/"; - } - - private void runInDockerContainer(ExtensionContext context) { - compose = createDockerCompose(context); - compose.start(); - - address = HostAndPort.fromParts( - browserHost(), compose.getServicePort(serviceName, DOCKER_PORT)); - rootPath = "/"; - } - - private String browserHost() { - return LOCAL_BROWSER ? "localhost" : "host.testcontainers.internal"; - } - - private void setBrowserContainerByOsName() { - DockerImageName imageName; - - if (M1_CHIP_FLAG) { - imageName = DockerImageName.parse("seleniarm/standalone-chromium:124.0-chromedriver-124.0") - .asCompatibleSubstituteFor("selenium/standalone-chrome"); - - browser = new BrowserWebDriverContainer<>(imageName) - .withCapabilities(new ChromeOptions()) - .withCreateContainerCmdModifier(cmd -> cmd.withUser("root")) - .withFileSystemBind( - Constants.HOST_CHROME_DOWNLOAD_PATH.toFile().getAbsolutePath(), - Constants.SELENIUM_CONTAINER_CHROME_DOWNLOAD_PATH) - .withRecordingMode(RECORD_ALL, record.toFile(), MP4) - .withStartupTimeout(Duration.ofSeconds(300)); - } else { - browser = new BrowserWebDriverContainer<>() - .withCapabilities(new ChromeOptions()) - .withCreateContainerCmdModifier(cmd -> cmd.withUser("root")) - .withFileSystemBind( - Constants.HOST_CHROME_DOWNLOAD_PATH.toFile().getAbsolutePath(), - Constants.SELENIUM_CONTAINER_CHROME_DOWNLOAD_PATH) - .withRecordingMode(RECORD_ALL, record.toFile(), MP4) - .withStartupTimeout(Duration.ofSeconds(300)); - } - } - - private void setRecordPath() throws IOException { - if (!Strings.isNullOrEmpty(System.getenv("RECORDING_PATH"))) { - record = Paths.get(System.getenv("RECORDING_PATH")); - if (!record.toFile().exists()) { - if (!record.toFile().mkdir()) { - throw new IOException("Failed to create recording directory: " + record.toAbsolutePath()); - } - } - } else { - record = Files.createTempDirectory("record-"); - } - } - - @Override - public void afterAll(ExtensionContext context) { - if (driver != null) { - driver.quit(); - } - if (!LOCAL_BROWSER && browser != null) { - browser.afterTest(new TestDescription(context), Optional.empty()); - browser.stop(); - } - if (compose != null) { - compose.stop(); - } - } - - @Override - public void beforeEach(ExtensionContext context) { - final Object instance = context.getRequiredTestInstance(); - Stream.of(instance.getClass().getDeclaredFields()) - .filter(f -> WebDriver.class.isAssignableFrom(f.getType())) - .forEach(it -> setDriver(instance, it)); - } - - private void setDriver(Object object, Field field) { - try { - field.setAccessible(true); - field.set(object, driver); - } catch (IllegalAccessException e) { - log.error("Failed to inject web driver to field: {}", field.getName(), e); - } - } - - private ComposeContainer createDockerCompose(ExtensionContext context) { - final Class clazz = context.getRequiredTestClass(); - final StreamPark annotation = clazz.getAnnotation(StreamPark.class); - final List files = Stream.of(annotation.composeFiles()) - .map(it -> StreamPark.class.getClassLoader().getResource(it)) - .filter(Objects::nonNull) - .map(URL::getPath) - .map(File::new) - .collect(Collectors.toList()); - - ComposeContainer compose = new ComposeContainer(files) - .withPull(true) - .withTailChildContainers(true) - .withLocalCompose(true) - .withExposedService( - serviceName, - DOCKER_PORT, - Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(300))) - .withLogConsumer(serviceName, outputFrame -> log.info(outputFrame.getUtf8String())) - .waitingFor( - serviceName, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300))); - - return compose; - } -} diff --git a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/TestDescription.java b/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/TestDescription.java deleted file mode 100644 index e19e51f073..0000000000 --- a/streampark-e2e/streampark-e2e-core/src/main/java/org/apache/streampark/e2e/core/TestDescription.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.e2e.core; - -import lombok.RequiredArgsConstructor; -import org.junit.jupiter.api.extension.ExtensionContext; - -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; - -import static java.nio.charset.StandardCharsets.UTF_8; - -@RequiredArgsConstructor -final class TestDescription implements org.testcontainers.lifecycle.TestDescription { - - private static final String UNKNOWN_NAME = "unknown"; - - private final ExtensionContext context; - - @Override - public String getTestId() { - return context.getUniqueId(); - } - - @Override - public String getFilesystemFriendlyName() { - final String contextId = context.getUniqueId(); - try { - return (contextId == null || contextId.trim().isEmpty()) - ? UNKNOWN_NAME - : URLEncoder.encode(contextId, UTF_8.toString()); - } catch (UnsupportedEncodingException e) { - return UNKNOWN_NAME; - } - } -} diff --git a/streampark-e2e/streampark-e2e-core/src/main/resources/docker-java.properties b/streampark-e2e/streampark-e2e-core/src/main/resources/docker-java.properties deleted file mode 100644 index 74f51e401a..0000000000 --- a/streampark-e2e/streampark-e2e-core/src/main/resources/docker-java.properties +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Docker Engine 29+ requires API version 1.44 or later. -# Testcontainers 1.19.x defaults to 1.32 via docker-java, which is rejected by the daemon. -api.version=1.44 diff --git a/streampark-e2e/streampark-e2e-core/src/main/resources/log4j2.xml b/streampark-e2e/streampark-e2e-core/src/main/resources/log4j2.xml deleted file mode 100644 index 167e6e6d97..0000000000 --- a/streampark-e2e/streampark-e2e-core/src/main/resources/log4j2.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/streampark-flink/pom.xml b/streampark-flink/pom.xml index c84c942ab5..d1def48ad5 100644 --- a/streampark-flink/pom.xml +++ b/streampark-flink/pom.xml @@ -31,9 +31,7 @@ streampark-flink-shims streampark-flink-sqlclient - streampark-flink-udf streampark-flink-client - streampark-flink-proxy streampark-flink-packer streampark-flink-kubernetes diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml b/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml index a3442949d2..b5af70844b 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/pom.xml @@ -35,43 +35,12 @@ provided - - org.apache.streampark - streampark-flink-proxy - ${project.version} - - org.apache.streampark streampark-flink-packer ${project.version} - - org.apache.streampark - streampark-flink-kubernetes - ${project.version} - provided - - - - org.apache.streampark - streampark-flink-shims-base - ${project.version} - provided - - - - org.apache.hadoop - hadoop-client-api - provided - - - - org.apache.hadoop - hadoop-client-runtime - provided - diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java index e014f26a87..eb0f9da30a 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java @@ -17,46 +17,34 @@ package org.apache.streampark.flink.client; -import org.apache.streampark.common.conf.FlinkVersion; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.DeployRequest; -import org.apache.streampark.flink.client.bean.DeployResponse; -import org.apache.streampark.flink.client.bean.SavepointResponse; -import org.apache.streampark.flink.client.bean.ShutDownRequest; -import org.apache.streampark.flink.client.bean.ShutDownResponse; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; +import org.apache.streampark.common.core.FlinkVersion; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.DeployRequest; +import org.apache.streampark.flink.client.request.ShutdownRequest; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.DeployResponse; +import org.apache.streampark.flink.client.response.SavepointResponse; +import org.apache.streampark.flink.client.response.ShutdownResponse; +import org.apache.streampark.flink.client.response.SubmitResponse; import org.apache.streampark.flink.proxy.FlinkShimsProxy; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.security.Permission; +/** Public facade for version-isolated Flink client operations. */ public final class FlinkClient { private static final String FLINK_CLIENT_ENTRYPOINT_CLASS = "org.apache.streampark.flink.client.FlinkClientEntrypoint"; - private static final String SUBMIT_REQUEST = - "org.apache.streampark.flink.client.bean.SubmitRequest"; - - private static final String DEPLOY_REQUEST = - "org.apache.streampark.flink.client.bean.DeployRequest"; - - private static final String CANCEL_REQUEST = - "org.apache.streampark.flink.client.bean.CancelRequest"; - - private static final String SHUTDOWN_REQUEST = - "org.apache.streampark.flink.client.bean.ShutDownRequest"; - - private static final String SAVEPOINT_REQUEST = - "org.apache.streampark.flink.client.bean.TriggerSavepointRequest"; - private FlinkClient() { } + /** Submits a Flink job using the version and deployment mode in the request. */ public static SubmitResponse submit(SubmitRequest submitRequest) { SecurityManager previousSecurityManager = System.getSecurityManager(); boolean exitGuardInstalled = false; @@ -70,7 +58,7 @@ public static SubmitResponse submit(SubmitRequest submitRequest) { return invokeClient( submitRequest, submitRequest.flinkVersion(), - SUBMIT_REQUEST, + SubmitRequest.class, "submit", SubmitResponse.class); } finally { @@ -80,30 +68,42 @@ public static SubmitResponse submit(SubmitRequest submitRequest) { } } + /** Cancels a running Flink job. */ public static CancelResponse cancel(CancelRequest stopRequest) { return invokeClient( - stopRequest, stopRequest.flinkVersion(), CANCEL_REQUEST, "cancel", CancelResponse.class); + stopRequest, + stopRequest.flinkVersion(), + CancelRequest.class, + "cancel", + CancelResponse.class); } + /** Deploys a Flink session cluster. */ public static DeployResponse deploy(DeployRequest deployRequest) { return invokeClient( - deployRequest, deployRequest.flinkVersion(), DEPLOY_REQUEST, "deploy", DeployResponse.class); + deployRequest, + deployRequest.flinkVersion(), + DeployRequest.class, + "deploy", + DeployResponse.class); } - public static ShutDownResponse shutdown(ShutDownRequest shutDownRequest) { + /** Shuts down a Flink session cluster. */ + public static ShutdownResponse shutdown(ShutdownRequest shutdownRequest) { return invokeClient( - shutDownRequest, - shutDownRequest.flinkVersion(), - SHUTDOWN_REQUEST, + shutdownRequest, + shutdownRequest.flinkVersion(), + ShutdownRequest.class, "shutdown", - ShutDownResponse.class); + ShutdownResponse.class); } + /** Triggers a savepoint for a running Flink job. */ public static SavepointResponse triggerSavepoint(TriggerSavepointRequest savepointRequest) { return invokeClient( savepointRequest, savepointRequest.flinkVersion(), - SAVEPOINT_REQUEST, + TriggerSavepointRequest.class, "triggerSavepoint", SavepointResponse.class); } @@ -111,7 +111,7 @@ public static SavepointResponse triggerSavepoint(TriggerSavepointRequest savepoi private static R invokeClient( Object request, FlinkVersion flinkVersion, - String requestClassName, + Class requestType, String methodName, Class responseType) { flinkVersion.checkVersion(); @@ -120,7 +120,7 @@ private static R invokeClient( classLoader -> { try { Class entrypointClass = classLoader.loadClass(FLINK_CLIENT_ENTRYPOINT_CLASS); - Class requestClass = classLoader.loadClass(requestClassName); + Class requestClass = classLoader.loadClass(requestType.getName()); Method method = entrypointClass.getDeclaredMethod(methodName, requestClass); method.setAccessible(true); Object shimsRequest = @@ -150,4 +150,26 @@ private static RuntimeException unwrapInvocationTarget(InvocationTargetException } return new IllegalStateException("Failed to invoke Flink client method", e); } + + /** Prevents submitted user code from terminating the console JVM. */ + static final class ExitSecurityManager extends SecurityManager { + + @Override + public void checkExit(int status) { + throw new SecurityException( + "System.exit(" + + status + + ") was called in your Flink job; the job has been stopped"); + } + + @Override + public void checkPermission(Permission permission) { + // Allow all operations except System.exit. + } + + @Override + public void checkPermission(Permission permission, Object context) { + // Allow all operations except System.exit. + } + } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequestTrait.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequestTrait.java deleted file mode 100644 index e48d51a7d6..0000000000 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequestTrait.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.client.bean; - -import org.apache.streampark.common.conf.FlinkVersion; -import org.apache.streampark.common.enums.FlinkDeployMode; - -import javax.annotation.Nullable; - -import java.util.Map; - -public interface DeployRequestTrait { - - FlinkVersion flinkVersion(); - - FlinkDeployMode deployMode(); - - Map properties(); - - String clusterId(); - - long id(); - - @Nullable - KubernetesDeployParam k8sParam(); -} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/HdfsWorkspace.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/HdfsWorkspace.java deleted file mode 100644 index bbe328b5a2..0000000000 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/HdfsWorkspace.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.client.bean; - -import java.io.Serializable; - -public class HdfsWorkspace implements Serializable { - - private static final long serialVersionUID = 1L; - - private final String flinkName; - private final String flinkHome; - private final String flinkDistJar; - private final String flinkLib; - private final String flinkPlugins; - private final String appJars; - - public HdfsWorkspace( - String flinkName, - String flinkHome, - String flinkDistJar, - String flinkLib, - String flinkPlugins, - String appJars) { - this.flinkName = flinkName; - this.flinkHome = flinkHome; - this.flinkDistJar = flinkDistJar; - this.flinkLib = flinkLib; - this.flinkPlugins = flinkPlugins; - this.appJars = appJars; - } - - public String flinkName() { - return flinkName; - } - - public String flinkHome() { - return flinkHome; - } - - public String flinkDistJar() { - return flinkDistJar; - } - - public String flinkLib() { - return flinkLib; - } - - public String flinkPlugins() { - return flinkPlugins; - } - - public String appJars() { - return appJars; - } -} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointCancelOptions.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointCancelOptions.java deleted file mode 100644 index baae013f4c..0000000000 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointCancelOptions.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.client.bean; - -import java.io.Serializable; - -/** Savepoint-related options for job cancel requests. */ -public final class SavepointCancelOptions implements Serializable { - - private static final long serialVersionUID = 1L; - - private final boolean withSavepoint; - private final boolean withDrain; - private final String savepointPath; - private final boolean nativeFormat; - - public SavepointCancelOptions( - boolean withSavepoint, - boolean withDrain, - String savepointPath, - boolean nativeFormat) { - this.withSavepoint = withSavepoint; - this.withDrain = withDrain; - this.savepointPath = savepointPath; - this.nativeFormat = nativeFormat; - } - - public boolean withSavepoint() { - return withSavepoint; - } - - public boolean withDrain() { - return withDrain; - } - - public String savepointPath() { - return savepointPath; - } - - public boolean nativeFormat() { - return nativeFormat; - } -} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointRequestTrait.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointRequestTrait.java deleted file mode 100644 index 219a485812..0000000000 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointRequestTrait.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.client.bean; - -import org.apache.streampark.common.conf.FlinkVersion; -import org.apache.streampark.common.constants.Constants; -import org.apache.streampark.common.enums.FlinkDeployMode; - -import javax.annotation.Nullable; - -import java.util.Map; - -public interface SavepointRequestTrait { - - FlinkVersion flinkVersion(); - - FlinkDeployMode deployMode(); - - String clusterId(); - - String jobId(); - - default boolean withSavepoint() { - return true; - } - - String savepointPath(); - - boolean nativeFormat(); - - default String kubernetesNamespace() { - return Constants.DEFAULT; - } - - @Nullable - Map properties(); -} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointTriggerOptions.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointTriggerOptions.java deleted file mode 100644 index e22c99f1af..0000000000 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointTriggerOptions.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.client.bean; - -import java.io.Serializable; - -/** Savepoint trigger options for savepoint requests. */ -public final class SavepointTriggerOptions implements Serializable { - - private static final long serialVersionUID = 1L; - - private final String savepointPath; - private final boolean nativeFormat; - - public SavepointTriggerOptions(String savepointPath, boolean nativeFormat) { - this.savepointPath = savepointPath; - this.nativeFormat = nativeFormat; - } - - public String savepointPath() { - return savepointPath; - } - - public boolean nativeFormat() { - return nativeFormat; - } -} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequest.java deleted file mode 100644 index 598f5e16ca..0000000000 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequest.java +++ /dev/null @@ -1,469 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.client.bean; - -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.conf.FlinkVersion; -import org.apache.streampark.common.conf.Workspace; -import org.apache.streampark.common.constants.Constants; -import org.apache.streampark.common.enums.ApplicationType; -import org.apache.streampark.common.enums.FlinkDeployMode; -import org.apache.streampark.common.enums.FlinkJobType; -import org.apache.streampark.common.enums.FlinkK8sRestExposedType; -import org.apache.streampark.common.enums.FlinkRestoreMode; -import org.apache.streampark.common.util.AssertUtils; -import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.HdfsUtils; -import org.apache.streampark.common.util.PropertiesUtils; -import org.apache.streampark.flink.client.conf.FlinkSavepointOptions; -import org.apache.streampark.flink.packer.pipeline.BuildResult; -import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse; - -import org.apache.streampark.shaded.com.fasterxml.jackson.core.type.TypeReference; -import org.apache.streampark.shaded.com.fasterxml.jackson.databind.ObjectMapper; - -import org.apache.commons.collections.MapUtils; -import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; - -import javax.annotation.Nullable; - -import java.io.File; -import java.io.Serializable; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class SubmitRequest implements Serializable { - - private static final long serialVersionUID = 1L; - - private final FlinkVersion flinkVersion; - private final FlinkDeployMode deployMode; - private final Map properties; - private final SubmitApplicationSpec application; - @Nullable - private final SubmitClusterSpec cluster; - @Nullable - private final BuildResult buildResult; - @Nullable - private final Map extraParameter; - - private transient Map appProperties; - private transient Map appOption; - private transient String appMain; - private transient String effectiveAppName; - private transient List libs; - private transient List classPaths; - private transient String flinkSQL; - private transient Boolean allowNonRestoredState; - private transient SavepointRestoreSettings savepointRestoreSettings; - private transient File userJarFile; - private transient Boolean safePackageProgram; - private transient HdfsWorkspace hdfsWorkspace; - - public SubmitRequest( - FlinkVersion flinkVersion, - FlinkDeployMode deployMode, - Map properties, - SubmitApplicationSpec application, - @Nullable SubmitClusterSpec cluster, - @Nullable BuildResult buildResult, - @Nullable Map extraParameter) { - this.flinkVersion = flinkVersion; - this.deployMode = deployMode; - this.properties = ClientBeanUtils.toSerializableMap(properties); - this.application = application; - this.cluster = cluster; - this.buildResult = buildResult; - this.extraParameter = ClientBeanUtils.toSerializableMap(extraParameter); - } - - public FlinkVersion flinkVersion() { - return flinkVersion; - } - - public FlinkDeployMode deployMode() { - return deployMode; - } - - public Map properties() { - return ClientBeanUtils.copyPropertiesMap(properties); - } - - public String flinkYaml() { - return application.flinkYaml(); - } - - public FlinkJobType jobType() { - return application.jobType(); - } - - public long id() { - return application.id(); - } - - public String jobId() { - return application.jobId(); - } - - public String appName() { - return application.appName(); - } - - public String appConf() { - return application.appConf(); - } - - public ApplicationType applicationType() { - return application.applicationType(); - } - - public String savePoint() { - return application.savePoint(); - } - - public FlinkRestoreMode restoreMode() { - return application.restoreMode(); - } - - public String args() { - return application.args(); - } - - @Nullable - public String clusterId() { - return cluster != null ? cluster.clusterId() : null; - } - - @Nullable - public String hadoopUser() { - return cluster != null ? cluster.hadoopUser() : null; - } - - @Nullable - public BuildResult buildResult() { - return buildResult; - } - - @Nullable - public Map extraParameter() { - return ClientBeanUtils.copyPropertiesMap(extraParameter); - } - - @Nullable - public String kubernetesNamespace() { - return cluster != null ? cluster.kubernetesNamespace() : null; - } - - @Nullable - public FlinkK8sRestExposedType flinkRestExposedType() { - return cluster != null ? cluster.flinkRestExposedType() : null; - } - - public Map appProperties() { - if (appProperties == null) { - appProperties = getParameterMap(ConfigKeys.KEY_FLINK_PROPERTY_PREFIX()); - } - return appProperties; - } - - public Map appOption() { - if (appOption == null) { - appOption = getParameterMap(ConfigKeys.KEY_FLINK_OPTION_PREFIX()); - } - return appOption; - } - - public String appMain() { - if (appMain == null) { - if (jobType() == FlinkJobType.FLINK_SQL) { - appMain = Constants.STREAMPARK_FLINKSQL_CLIENT_CLASS; - } else if (jobType() == FlinkJobType.PYFLINK) { - appMain = Constants.PYTHON_FLINK_DRIVER_CLASS_NAME; - } else { - appMain = appProperties().get(ConfigKeys.KEY_FLINK_APPLICATION_MAIN_CLASS()); - if (appMain == null && appConf() != null) { - String format = appConf().substring(0, Math.min(appConf().length(), 7)); - appMain = loadAppConfMap(format).get(ConfigKeys.KEY_FLINK_APPLICATION_MAIN_CLASS()); - } - } - } - return appMain; - } - - public String effectiveAppName() { - if (effectiveAppName == null) { - effectiveAppName = - appName() == null ? appProperties().get(ConfigKeys.KEY_FLINK_APP_NAME()) : appName(); - } - return effectiveAppName; - } - - public List libs() { - if (libs == null) { - File libDir = new File(new File(Workspace.local().APP_WORKSPACE(), String.valueOf(id())), "lib"); - File[] files = libDir.listFiles(); - if (files == null) { - libs = Collections.emptyList(); - } else { - List urls = new ArrayList<>(); - for (File file : files) { - try { - urls.add(file.toURI().toURL()); - } catch (MalformedURLException e) { - throw new IllegalArgumentException("Invalid lib file URL: " + file, e); - } - } - libs = urls; - } - } - return libs; - } - - public List classPaths() { - if (classPaths == null) { - try { - List combined = new ArrayList<>(flinkVersion.getFlinkLibs()); - combined.addAll(libs()); - classPaths = combined; - } catch (Exception e) { - throw new IllegalStateException("Failed to build classpath", e); - } - } - return classPaths; - } - - public String flinkSQL() { - if (flinkSQL == null) { - if (extraParameter == null) { - return null; - } - Object sql = extraParameter.get(ConfigKeys.KEY_FLINK_SQL()); - flinkSQL = sql != null ? sql.toString() : null; - } - return flinkSQL; - } - - public boolean allowNonRestoredState() { - if (allowNonRestoredState == null) { - Object value = - properties.get(FlinkSavepointOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE.key()); - if (value == null) { - allowNonRestoredState = false; - } else { - try { - allowNonRestoredState = Boolean.parseBoolean(value.toString()); - } catch (Exception e) { - allowNonRestoredState = false; - } - } - } - return allowNonRestoredState; - } - - public SavepointRestoreSettings savepointRestoreSettings() { - if (savepointRestoreSettings == null) { - if (savePoint() == null || savePoint().isEmpty()) { - savepointRestoreSettings = SavepointRestoreSettings.none(); - } else { - savepointRestoreSettings = - SavepointRestoreSettings.forPath(savePoint(), allowNonRestoredState()); - } - } - return savepointRestoreSettings; - } - - public File userJarFile() { - if (userJarFile == null) { - if (deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) { - userJarFile = null; - } else { - checkBuildResult(); - ShadedBuildResponse shadedBuildResult = buildResult.as(ShadedBuildResponse.class); - String shadedJarPath = shadedBuildResult.shadedJarPath(); - userJarFile = shadedJarPath == null ? null : new File(shadedJarPath); - } - } - return userJarFile; - } - - public boolean safePackageProgram() { - if (safePackageProgram == null) { - String version = flinkVersion.version(); - String[] parts = version.split("\\."); - if (parts.length >= 3) { - try { - int a = Integer.parseInt(parts[0].trim()); - int b = Integer.parseInt(parts[1].trim()); - int c = Integer.parseInt(parts[2].trim()); - safePackageProgram = a >= 1 && (b > 12 || (b == 12 && c >= 2)); - } catch (NumberFormatException e) { - safePackageProgram = false; - } - } else { - safePackageProgram = false; - } - } - return safePackageProgram; - } - - public boolean hasProp(String key) { - return MapUtils.isNotEmpty(properties) && properties.containsKey(key); - } - - public Object getProp(String key) { - return properties.get(key); - } - - public void putProperty(String key, Serializable value) { - if (properties != null) { - properties.put(key, value); - } - } - - public boolean hasExtra(String key) { - return MapUtils.isNotEmpty(extraParameter) && extraParameter.containsKey(key); - } - - public Object getExtra(String key) { - return extraParameter != null ? extraParameter.get(key) : null; - } - - public HdfsWorkspace hdfsWorkspace() { - if (hdfsWorkspace == null) { - hdfsWorkspace = ClientBeanUtils.createHdfsWorkspace(flinkVersion); - } - return hdfsWorkspace; - } - - public void checkBuildResult() { - if (deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_SESSION) { - AssertUtils.required( - buildResult != null, - "[flink-submit] current job: " - + effectiveAppName() - + " was not yet built, buildResult is empty" - + ",clusterId=" - + clusterId() - + "," - + ",namespace=" - + kubernetesNamespace()); - AssertUtils.required( - buildResult.pass(), - "[flink-submit] current job " - + effectiveAppName() - + " build failed, clusterId" - + ",clusterId=" - + clusterId() - + "," - + ",namespace=" - + kubernetesNamespace()); - } else { - AssertUtils.required( - buildResult != null, - "[flink-submit] current job: " - + effectiveAppName() - + " was not yet built, buildResult is empty"); - AssertUtils.required( - buildResult.pass(), - "[flink-submit] current job " + effectiveAppName() + " build failed, please check"); - } - } - - private Map getParameterMap(String prefix) { - if (appConf() == null) { - return Collections.emptyMap(); - } - String format = appConf().substring(0, Math.min(appConf().length(), 7)); - Map map = loadAppConfMap(format); - return filterByPrefix(map, prefix); - } - - private Map loadAppConfMap(String format) { - if ("json://".equals(format)) { - return parseJsonAppConf(); - } - String content = DeflaterUtils.unzipString(appConf().trim().substring(7)); - switch (format) { - case "yaml://": - return PropertiesUtils.fromYamlTextAsJava(content); - case "conf://": - return PropertiesUtils.fromHoconTextAsJava(content); - case "prop://": - return PropertiesUtils.fromPropertiesTextAsJava(content); - case "hdfs://": - return parseHdfsAppConf(); - default: - throw new IllegalArgumentException("[StreamPark] application config format error."); - } - } - - private Map parseJsonAppConf() { - String json = appConf().substring(7); - try { - Map map = - new ObjectMapper() - .readValue(json, new TypeReference>() { - }); - return map.entrySet().stream() - .filter(e -> e.getValue() != null) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } catch (Exception e) { - throw new IllegalArgumentException("Failed to parse JSON app config", e); - } - } - - private Map parseHdfsAppConf() { - try { - String text = HdfsUtils.read(appConf()); - String[] parts = appConf().split("\\."); - String extension = parts[parts.length - 1].toLowerCase(); - switch (extension) { - case "yml": - case "yaml": - return PropertiesUtils.fromYamlTextAsJava(text); - case "conf": - return PropertiesUtils.fromHoconTextAsJava(text); - case "properties": - return PropertiesUtils.fromPropertiesTextAsJava(text); - default: - throw new IllegalArgumentException( - "[StreamPark] Usage: application config format error,must be [yaml|conf|properties]"); - } - } catch (Exception e) { - throw new IllegalStateException("Failed to parse HDFS app config: " + appConf(), e); - } - } - - private static Map filterByPrefix(Map map, String prefix) { - Map result = new HashMap<>(); - for (Map.Entry entry : map.entrySet()) { - String key = entry.getKey(); - String value = entry.getValue(); - if (key.startsWith(prefix) && value != null && !value.isEmpty()) { - result.put(key.substring(prefix.length()), value); - } - } - return result; - } -} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/CancelRequest.java similarity index 63% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelRequest.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/CancelRequest.java index 02d47158cf..d3f143e8b6 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/CancelRequest.java @@ -15,20 +15,24 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.request; -import org.apache.streampark.common.conf.FlinkVersion; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import javax.annotation.Nullable; import java.util.Map; -public class CancelRequest extends AbstractSavepointClientRequest { +/** Request to cancel a Flink job, optionally creating a savepoint first. */ +public final class CancelRequest extends SavepointRequest { private static final long serialVersionUID = 1L; - private final SavepointCancelOptions savepointOptions; + private final boolean withSavepoint; + private final boolean withDrain; + private final String savepointPath; + private final boolean nativeFormat; public CancelRequest( long id, @@ -36,27 +40,33 @@ public CancelRequest( FlinkDeployMode deployMode, @Nullable Map properties, JobClientTarget target, - SavepointCancelOptions savepointOptions) { + boolean withSavepoint, + boolean withDrain, + @Nullable String savepointPath, + boolean nativeFormat) { super(id, flinkVersion, deployMode, properties, target); - this.savepointOptions = savepointOptions; + this.withSavepoint = withSavepoint; + this.withDrain = withDrain; + this.savepointPath = savepointPath; + this.nativeFormat = nativeFormat; } @Override public boolean withSavepoint() { - return savepointOptions.withSavepoint(); + return withSavepoint; } public boolean withDrain() { - return savepointOptions.withDrain(); + return withDrain; } @Override public String savepointPath() { - return savepointOptions.savepointPath(); + return savepointPath; } @Override public boolean nativeFormat() { - return savepointOptions.nativeFormat(); + return nativeFormat; } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClientRequestUtils.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClientRequestUtils.java new file mode 100644 index 0000000000..c7d021376d --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClientRequestUtils.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.request; + +import javax.annotation.Nullable; + +import java.io.Serializable; +import java.util.LinkedHashMap; +import java.util.Map; + +/** Defensive map conversions shared by immutable Flink client request objects. */ +final class ClientRequestUtils { + + private ClientRequestUtils() { + } + + static Map copyPropertiesMap( + @Nullable Map properties) { + return properties == null ? new LinkedHashMap<>() : new LinkedHashMap<>(properties); + } + + static Map toSerializableMap( + @Nullable Map properties) { + Map result = new LinkedHashMap<>(); + if (properties == null) { + return result; + } + for (Map.Entry entry : properties.entrySet()) { + Object value = entry.getValue(); + if (value instanceof Serializable) { + result.put(entry.getKey(), (Serializable) value); + } else if (value != null) { + result.put(entry.getKey(), value.toString()); + } + } + return result; + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractDeployClientRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClusterRequest.java similarity index 62% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractDeployClientRequest.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClusterRequest.java index 53b2b901eb..8a5665594b 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractDeployClientRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClusterRequest.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.request; -import org.apache.streampark.common.conf.FlinkVersion; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import javax.annotation.Nullable; @@ -25,64 +25,56 @@ import java.io.Serializable; import java.util.Map; -/** Shared fields for deploy and shutdown client requests. */ -abstract class AbstractDeployClientRequest implements DeployRequestTrait, Serializable { +/** Base request for operations on a Flink session cluster. */ +public abstract class ClusterRequest implements Serializable { private static final long serialVersionUID = 1L; private final FlinkVersion flinkVersion; private final FlinkDeployMode deployMode; - @Nullable private final Map properties; private final String clusterId; private final long id; @Nullable - private final KubernetesDeployParam k8sParam; + private final KubernetesDeploySpec kubernetesDeploySpec; - AbstractDeployClientRequest( - FlinkVersion flinkVersion, - FlinkDeployMode deployMode, - @Nullable Map properties, - String clusterId, - long id, - @Nullable KubernetesDeployParam k8sParam) { + protected ClusterRequest( + FlinkVersion flinkVersion, + FlinkDeployMode deployMode, + @Nullable Map properties, + String clusterId, + long id, + @Nullable KubernetesDeploySpec kubernetesDeploySpec) { this.flinkVersion = flinkVersion; this.deployMode = deployMode; - this.properties = ClientBeanUtils.toSerializableMap(properties); + this.properties = ClientRequestUtils.toSerializableMap(properties); this.clusterId = clusterId; this.id = id; - this.k8sParam = k8sParam; + this.kubernetesDeploySpec = kubernetesDeploySpec; } - @Override public FlinkVersion flinkVersion() { return flinkVersion; } - @Override public FlinkDeployMode deployMode() { return deployMode; } - @Override - @Nullable public Map properties() { - return ClientBeanUtils.copyPropertiesMap(properties); + return ClientRequestUtils.copyPropertiesMap(properties); } - @Override public String clusterId() { return clusterId; } - @Override public long id() { return id; } - @Override @Nullable - public KubernetesDeployParam k8sParam() { - return k8sParam; + public KubernetesDeploySpec kubernetesDeploySpec() { + return kubernetesDeploySpec; } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/DeployRequest.java similarity index 72% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequest.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/DeployRequest.java index d4a4505a54..20611e64e8 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/DeployRequest.java @@ -15,35 +15,28 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.request; -import org.apache.streampark.common.conf.FlinkVersion; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import javax.annotation.Nullable; import java.util.Map; -public class DeployRequest extends AbstractDeployClientRequest { +/** Request to deploy a Flink session cluster. */ +public final class DeployRequest extends ClusterRequest { private static final long serialVersionUID = 1L; - private transient HdfsWorkspace hdfsWorkspace; - public DeployRequest( FlinkVersion flinkVersion, FlinkDeployMode deployMode, Map properties, String clusterId, long id, - @Nullable KubernetesDeployParam k8sParam) { - super(flinkVersion, deployMode, properties, clusterId, id, k8sParam); + @Nullable KubernetesDeploySpec kubernetesDeploySpec) { + super(flinkVersion, deployMode, properties, clusterId, id, kubernetesDeploySpec); } - public HdfsWorkspace hdfsWorkspace() { - if (hdfsWorkspace == null) { - hdfsWorkspace = ClientBeanUtils.createHdfsWorkspace(flinkVersion()); - } - return hdfsWorkspace; - } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/JobClientTarget.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/JobClientTarget.java similarity index 93% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/JobClientTarget.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/JobClientTarget.java index 7bca4278a2..43f3798a0b 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/JobClientTarget.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/JobClientTarget.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.request; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; import java.io.Serializable; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/KubernetesDeployParam.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/KubernetesDeploySpec.java similarity index 60% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/KubernetesDeployParam.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/KubernetesDeploySpec.java index 26c11cf163..dcde4ebef2 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/KubernetesDeployParam.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/KubernetesDeploySpec.java @@ -15,17 +15,16 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.request; -import org.apache.streampark.common.enums.FlinkK8sRestExposedType; - -import org.apache.flink.kubernetes.configuration.KubernetesConfigOptions; +import org.apache.streampark.common.enums.FlinkKubernetesRestExposedType; import javax.annotation.Nullable; import java.io.Serializable; -public class KubernetesDeployParam implements Serializable { +/** Kubernetes-specific settings for session-cluster deployment. */ +public final class KubernetesDeploySpec implements Serializable { private static final long serialVersionUID = 1L; @@ -35,32 +34,22 @@ public class KubernetesDeployParam implements Serializable { private final String serviceAccount; private final String flinkImage; @Nullable - private final FlinkK8sRestExposedType flinkRestExposedType; - - public KubernetesDeployParam(String clusterId) { - this( - clusterId, - KubernetesConfigOptions.NAMESPACE.defaultValue(), - "~/.kube/config", - KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT.defaultValue(), - KubernetesConfigOptions.CONTAINER_IMAGE.defaultValue(), - FlinkK8sRestExposedType.CLUSTER_IP); - } - - public KubernetesDeployParam( - String clusterId, - String kubernetesNamespace, - String kubeConf, - String serviceAccount, - String flinkImage, - @Nullable FlinkK8sRestExposedType flinkRestExposedType) { + private final FlinkKubernetesRestExposedType flinkRestExposedType; + + public KubernetesDeploySpec( + String clusterId, + String kubernetesNamespace, + String kubeConf, + String serviceAccount, + String flinkImage, + @Nullable FlinkKubernetesRestExposedType flinkRestExposedType) { this.clusterId = clusterId; this.kubernetesNamespace = kubernetesNamespace; this.kubeConf = kubeConf; this.serviceAccount = serviceAccount; this.flinkImage = flinkImage; this.flinkRestExposedType = - flinkRestExposedType != null ? flinkRestExposedType : FlinkK8sRestExposedType.CLUSTER_IP; + flinkRestExposedType != null ? flinkRestExposedType : FlinkKubernetesRestExposedType.CLUSTER_IP; } public String clusterId() { @@ -84,7 +73,7 @@ public String flinkImage() { } @Nullable - public FlinkK8sRestExposedType flinkRestExposedType() { + public FlinkKubernetesRestExposedType flinkRestExposedType() { return flinkRestExposedType; } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractSavepointClientRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SavepointRequest.java similarity index 63% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractSavepointClientRequest.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SavepointRequest.java index ffe029b451..67419e419c 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/AbstractSavepointClientRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SavepointRequest.java @@ -15,9 +15,10 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.request; -import org.apache.streampark.common.conf.FlinkVersion; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import javax.annotation.Nullable; @@ -25,28 +26,27 @@ import java.io.Serializable; import java.util.Map; -/** Shared fields for cancel and trigger-savepoint client requests. */ -abstract class AbstractSavepointClientRequest implements SavepointRequestTrait, Serializable { +/** Base request for job operations that can create a savepoint. */ +public abstract class SavepointRequest implements Serializable { private static final long serialVersionUID = 1L; private final long id; private final FlinkVersion flinkVersion; private final FlinkDeployMode deployMode; - @Nullable private final Map properties; private final JobClientTarget target; - AbstractSavepointClientRequest( - long id, - FlinkVersion flinkVersion, - FlinkDeployMode deployMode, - @Nullable Map properties, - JobClientTarget target) { + protected SavepointRequest( + long id, + FlinkVersion flinkVersion, + FlinkDeployMode deployMode, + @Nullable Map properties, + JobClientTarget target) { this.id = id; this.flinkVersion = flinkVersion; this.deployMode = deployMode; - this.properties = ClientBeanUtils.toSerializableMap(properties); + this.properties = ClientRequestUtils.toSerializableMap(properties); this.target = target; } @@ -54,34 +54,36 @@ public long id() { return id; } - @Override public FlinkVersion flinkVersion() { return flinkVersion; } - @Override public FlinkDeployMode deployMode() { return deployMode; } - @Override - @Nullable public Map properties() { - return ClientBeanUtils.copyPropertiesMap(properties); + return ClientRequestUtils.copyPropertiesMap(properties); } - @Override public String clusterId() { return target.clusterId(); } - @Override public String jobId() { return target.jobId(); } - @Override + public boolean withSavepoint() { + return true; + } + + public abstract String savepointPath(); + + public abstract boolean nativeFormat(); + public String kubernetesNamespace() { - return target.kubernetesNamespace(); + String namespace = target.kubernetesNamespace(); + return namespace == null ? Constants.DEFAULT : namespace; } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ShutdownRequest.java similarity index 78% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownRequest.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ShutdownRequest.java index 4fc78d2234..59aaf57242 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ShutdownRequest.java @@ -15,26 +15,27 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.request; -import org.apache.streampark.common.conf.FlinkVersion; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import javax.annotation.Nullable; import java.util.Map; -public class ShutDownRequest extends AbstractDeployClientRequest { +/** Request to shut down a Flink session cluster. */ +public final class ShutdownRequest extends ClusterRequest { private static final long serialVersionUID = 1L; - public ShutDownRequest( + public ShutdownRequest( FlinkVersion flinkVersion, FlinkDeployMode deployMode, @Nullable Map properties, String clusterId, long id, - @Nullable KubernetesDeployParam k8sParam) { - super(flinkVersion, deployMode, properties, clusterId, id, k8sParam); + @Nullable KubernetesDeploySpec kubernetesDeploySpec) { + super(flinkVersion, deployMode, properties, clusterId, id, kubernetesDeploySpec); } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitApplicationSpec.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitApplicationSpec.java similarity index 88% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitApplicationSpec.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitApplicationSpec.java index 3729d9921d..f93877e060 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitApplicationSpec.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitApplicationSpec.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.request; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.FlinkJobType; @@ -29,6 +29,7 @@ public final class SubmitApplicationSpec implements Serializable { private static final long serialVersionUID = 1L; private final String flinkYaml; + private final boolean standardYaml; private final FlinkJobType jobType; private final long id; private final String jobId; @@ -41,6 +42,7 @@ public final class SubmitApplicationSpec implements Serializable { private SubmitApplicationSpec(Builder builder) { this.flinkYaml = builder.flinkYaml; + this.standardYaml = builder.standardYaml; this.jobType = builder.jobType; this.id = builder.id; this.jobId = builder.jobId; @@ -60,6 +62,11 @@ public String flinkYaml() { return flinkYaml; } + /** Returns whether the persisted Flink configuration uses standard nested YAML. */ + public boolean standardYaml() { + return standardYaml; + } + public FlinkJobType jobType() { return jobType; } @@ -99,6 +106,7 @@ public String args() { public static final class Builder { private String flinkYaml; + private boolean standardYaml; private FlinkJobType jobType; private long id; private String jobId; @@ -114,6 +122,12 @@ public Builder flinkYaml(String flinkYaml) { return this; } + /** Records the syntax selected when the Flink configuration content was persisted. */ + public Builder standardYaml(boolean standardYaml) { + this.standardYaml = standardYaml; + return this; + } + public Builder jobType(FlinkJobType jobType) { this.jobType = jobType; return this; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitClusterSpec.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitClusterSpec.java similarity index 84% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitClusterSpec.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitClusterSpec.java index a6ffd8f07a..0d1d4f2fce 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitClusterSpec.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitClusterSpec.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.request; -import org.apache.streampark.common.enums.FlinkK8sRestExposedType; +import org.apache.streampark.common.enums.FlinkKubernetesRestExposedType; import javax.annotation.Nullable; @@ -35,13 +35,13 @@ public final class SubmitClusterSpec implements Serializable { @Nullable private final String kubernetesNamespace; @Nullable - private final FlinkK8sRestExposedType flinkRestExposedType; + private final FlinkKubernetesRestExposedType flinkRestExposedType; public SubmitClusterSpec( @Nullable String clusterId, @Nullable String hadoopUser, @Nullable String kubernetesNamespace, - @Nullable FlinkK8sRestExposedType flinkRestExposedType) { + @Nullable FlinkKubernetesRestExposedType flinkRestExposedType) { this.clusterId = clusterId; this.hadoopUser = hadoopUser; this.kubernetesNamespace = kubernetesNamespace; @@ -64,7 +64,7 @@ public String kubernetesNamespace() { } @Nullable - public FlinkK8sRestExposedType flinkRestExposedType() { + public FlinkKubernetesRestExposedType flinkRestExposedType() { return flinkRestExposedType; } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitRequest.java new file mode 100644 index 0000000000..7eb959f1c4 --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitRequest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.request; + +import org.apache.streampark.common.core.FlinkVersion; +import org.apache.streampark.common.enums.ApplicationType; +import org.apache.streampark.common.enums.FlinkDeployMode; +import org.apache.streampark.common.enums.FlinkJobType; +import org.apache.streampark.common.enums.FlinkKubernetesRestExposedType; +import org.apache.streampark.common.enums.FlinkRestoreMode; +import org.apache.streampark.flink.packer.pipeline.BuildResult; + +import javax.annotation.Nullable; + +import java.io.Serializable; +import java.util.Map; + +/** Immutable input for submitting a Flink job. */ +public final class SubmitRequest implements Serializable { + + private static final long serialVersionUID = 1L; + + private final FlinkVersion flinkVersion; + private final FlinkDeployMode deployMode; + private final Map properties; + private final SubmitApplicationSpec application; + @Nullable + private final SubmitClusterSpec cluster; + @Nullable + private final BuildResult buildResult; + private final Map extraParameter; + + public SubmitRequest( + FlinkVersion flinkVersion, + FlinkDeployMode deployMode, + Map properties, + SubmitApplicationSpec application, + @Nullable SubmitClusterSpec cluster, + @Nullable BuildResult buildResult, + @Nullable Map extraParameter) { + this.flinkVersion = flinkVersion; + this.deployMode = deployMode; + this.properties = ClientRequestUtils.toSerializableMap(properties); + this.application = application; + this.cluster = cluster; + this.buildResult = buildResult; + this.extraParameter = ClientRequestUtils.toSerializableMap(extraParameter); + } + + public FlinkVersion flinkVersion() { + return flinkVersion; + } + + public FlinkDeployMode deployMode() { + return deployMode; + } + + public Map properties() { + return ClientRequestUtils.copyPropertiesMap(properties); + } + + public String flinkYaml() { + return application.flinkYaml(); + } + + /** Returns the YAML syntax captured when the Flink configuration was persisted. */ + public boolean standardYaml() { + return application.standardYaml(); + } + + public FlinkJobType jobType() { + return application.jobType(); + } + + public long id() { + return application.id(); + } + + public String jobId() { + return application.jobId(); + } + + public String appName() { + return application.appName(); + } + + public String appConf() { + return application.appConf(); + } + + public ApplicationType applicationType() { + return application.applicationType(); + } + + public String savePoint() { + return application.savePoint(); + } + + public FlinkRestoreMode restoreMode() { + return application.restoreMode(); + } + + public String args() { + return application.args(); + } + + @Nullable + public String clusterId() { + return cluster == null ? null : cluster.clusterId(); + } + + @Nullable + public String hadoopUser() { + return cluster == null ? null : cluster.hadoopUser(); + } + + @Nullable + public String kubernetesNamespace() { + return cluster == null ? null : cluster.kubernetesNamespace(); + } + + @Nullable + public FlinkKubernetesRestExposedType flinkRestExposedType() { + return cluster == null ? null : cluster.flinkRestExposedType(); + } + + @Nullable + public BuildResult buildResult() { + return buildResult; + } + + public Map extraParameter() { + return ClientRequestUtils.copyPropertiesMap(extraParameter); + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/TriggerSavepointRequest.java similarity index 74% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/TriggerSavepointRequest.java index 4b972d9cec..8e9644e2e1 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/TriggerSavepointRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/TriggerSavepointRequest.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.request; -import org.apache.streampark.common.conf.FlinkVersion; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import javax.annotation.Nullable; @@ -25,11 +25,12 @@ import java.util.Map; /** Trigger savepoint request. */ -public class TriggerSavepointRequest extends AbstractSavepointClientRequest { +public final class TriggerSavepointRequest extends SavepointRequest { private static final long serialVersionUID = 1L; - private final SavepointTriggerOptions savepointOptions; + private final String savepointPath; + private final boolean nativeFormat; public TriggerSavepointRequest( long id, @@ -37,18 +38,20 @@ public TriggerSavepointRequest( FlinkDeployMode deployMode, @Nullable Map properties, JobClientTarget target, - SavepointTriggerOptions savepointOptions) { + @Nullable String savepointPath, + boolean nativeFormat) { super(id, flinkVersion, deployMode, properties, target); - this.savepointOptions = savepointOptions; + this.savepointPath = savepointPath; + this.nativeFormat = nativeFormat; } @Override public String savepointPath() { - return savepointOptions.savepointPath(); + return savepointPath; } @Override public boolean nativeFormat() { - return savepointOptions.nativeFormat(); + return nativeFormat; } } diff --git a/streampark-flink/streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/Length.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/package-info.java similarity index 74% rename from streampark-flink/streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/Length.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/package-info.java index 11a297f198..e175ca4c15 100644 --- a/streampark-flink/streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/Length.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/package-info.java @@ -15,16 +15,5 @@ * limitations under the License. */ -package org.apache.streampark.flink.udf; - -import org.apache.flink.table.functions.ScalarFunction; - -public class Length extends ScalarFunction { - - public Integer eval(String str) { - if (str == null) { - return 0; - } - return str.length(); - } -} +/** Serializable request contracts and value objects for Flink client operations. */ +package org.apache.streampark.flink.client.request; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelResponse.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/CancelResponse.java similarity index 87% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelResponse.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/CancelResponse.java index 1960d1e5e9..24601c0d53 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/CancelResponse.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/CancelResponse.java @@ -15,11 +15,12 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.response; import java.io.Serializable; -public class CancelResponse implements Serializable { +/** Result of cancelling a Flink job. */ +public final class CancelResponse implements Serializable { private static final long serialVersionUID = 1L; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployResponse.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/DeployResponse.java similarity index 91% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployResponse.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/DeployResponse.java index 559b521804..d524b50876 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/DeployResponse.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/DeployResponse.java @@ -15,13 +15,14 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.response; import javax.annotation.Nullable; import java.io.Serializable; -public class DeployResponse implements Serializable { +/** Result of deploying a Flink session cluster. */ +public final class DeployResponse implements Serializable { private static final long serialVersionUID = 1L; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointResponse.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/SavepointResponse.java similarity index 86% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointResponse.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/SavepointResponse.java index 76d341da62..6229ce22c0 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SavepointResponse.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/SavepointResponse.java @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.response; import java.io.Serializable; -/** Result class of trigger savepoint presents savepoint path. */ -public class SavepointResponse implements Serializable { +/** Result of triggering a Flink savepoint. */ +public final class SavepointResponse implements Serializable { private static final long serialVersionUID = 1L; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownResponse.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/ShutdownResponse.java similarity index 82% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownResponse.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/ShutdownResponse.java index 736b4bbe91..c87d170743 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ShutDownResponse.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/ShutdownResponse.java @@ -15,17 +15,18 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.response; import java.io.Serializable; -public class ShutDownResponse implements Serializable { +/** Result of shutting down a Flink session cluster. */ +public final class ShutdownResponse implements Serializable { private static final long serialVersionUID = 1L; private final String clusterId; - public ShutDownResponse(String clusterId) { + public ShutdownResponse(String clusterId) { this.clusterId = clusterId; } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitResponse.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/SubmitResponse.java similarity index 87% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitResponse.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/SubmitResponse.java index 985ebc67c0..f293f62811 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/SubmitResponse.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/SubmitResponse.java @@ -15,14 +15,17 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.response; import javax.annotation.Nullable; import java.io.Serializable; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.Map; -public class SubmitResponse implements Serializable { +/** Result of submitting a Flink job. */ +public final class SubmitResponse implements Serializable { private static final long serialVersionUID = 1L; @@ -50,7 +53,8 @@ public SubmitResponse( @Nullable String jobId, @Nullable String jobManagerUrl) { this.clusterId = clusterId; - this.flinkConfig = flinkConfig; + this.flinkConfig = + Collections.unmodifiableMap(new LinkedHashMap<>(flinkConfig)); this.jobId = jobId != null ? jobId : ""; this.jobManagerUrl = jobManagerUrl != null ? jobManagerUrl : ""; } diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/entity/UserManagementStatus.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/package-info.java similarity index 87% rename from streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/entity/UserManagementStatus.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/package-info.java index da83575dc8..e56aa34231 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/entity/UserManagementStatus.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/response/package-info.java @@ -15,8 +15,5 @@ * limitations under the License. */ -package org.apache.streampark.e2e.pages.system.entity; - -public enum UserManagementStatus { - LOCKED, EFFECTIVE -} +/** Serializable results returned by Flink client operations. */ +package org.apache.streampark.flink.client.response; diff --git a/streampark-flink/streampark-flink-proxy/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java similarity index 75% rename from streampark-flink/streampark-flink-proxy/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java index 3ccd9a6423..5c8081fdca 100644 --- a/streampark-flink/streampark-flink-proxy/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java @@ -17,9 +17,9 @@ package org.apache.streampark.flink.proxy; -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.conf.FlinkVersion; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.option.CoreOptions; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.util.ChildFirstClassLoader; import org.apache.streampark.common.util.ClassLoaderObjectInputStream; import org.apache.streampark.common.util.ClassLoaderUtils; @@ -35,7 +35,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -44,7 +43,14 @@ import java.util.function.Predicate; import java.util.regex.Pattern; -/** Proxy for loading Flink version-specific shims behind an isolated classloader. */ +/** + * Executes version-specific Flink integration code behind an isolated classloader. + * + *

Both submission and SQL validation load the complete {@code lib} directory of the registered + * Flink installation. SQL validation additionally loads table artifacts from {@code opt}. Keeping + * the target runtime intact prevents table APIs from linking against Flink classes supplied by the + * Console parent classloader. + */ public final class FlinkShimsProxy extends LoggerSupport { private static final FlinkShimsProxy LOG = new FlinkShimsProxy(); @@ -63,8 +69,6 @@ public final class FlinkShimsProxy extends LoggerSupport { private static final String FLINK_SHIMS_BASE_PREFIX = "streampark-flink-shims-base"; - private static final String FLINK_SHIMS_BASE_V2_PREFIX = "streampark-flink-shims-base-v2"; - private static final String STREAMPARK_PREFIX = "streampark-"; private static final String STREAMPARK_CONSOLE_PREFIX = "streampark-console"; @@ -89,13 +93,6 @@ public final class FlinkShimsProxy extends LoggerSupport { private FlinkShimsProxy() { } - /** {@code majorVersion} is shaped like "1.20" or "2.2". */ - private static boolean isFlink2(String majorVersion) { - int dot = majorVersion.indexOf('.'); - String major = dot < 0 ? majorVersion : majorVersion.substring(0, dot); - return "2".equals(major); - } - private static Pattern getFlinkShimsResourcePattern(String majorVersion) { return Pattern.compile( "flink-(.*)-" + majorVersion + "(.*).jar", @@ -144,17 +141,12 @@ public static T getObject(ClassLoader loader, Object obj, } } - // need to load all flink-table dependencies compatible with different versions private static ClassLoader getVerifySqlLibClassLoader(FlinkVersion flinkVersion) { LOG.logInfo("Add verify sql lib,flink version: " + flinkVersion); return VERIFY_SQL_CLASS_LOADER_CACHE.computeIfAbsent( flinkVersion.fullVersion(), key -> { - Predicate getFlinkTable = f -> f.getName().startsWith("flink-table"); - List libTableURL = getFlinkHomeLib(flinkVersion.flinkHome, "lib", getFlinkTable); - List optTableURL = getFlinkHomeLib(flinkVersion.flinkHome, "opt", getFlinkTable); - List shimsUrls = new ArrayList<>(libTableURL); - shimsUrls.addAll(optTableURL); + List shimsUrls = getFlinkUrls(flinkVersion, true); addShimsUrls( flinkVersion, @@ -164,11 +156,7 @@ private static ClassLoader getVerifySqlLibClassLoader(FlinkVersion flinkVersion) } }); - return new ChildFirstClassLoader( - shimsUrls.toArray(new URL[0]), - Thread.currentThread().getContextClassLoader(), - PARENT_FIRST_PATTERNS, - jarName -> loadJarFilter(jarName, flinkVersion)); + return createClassLoader(flinkVersion, shimsUrls); }); } @@ -179,10 +167,10 @@ private static boolean loadJarFilter(String jarName, FlinkVersion flinkVersion) } private static void addShimsUrls(FlinkVersion flinkVersion, Consumer addShimUrl) { - String appHome = System.getProperty(ConfigKeys.KEY_APP_HOME()); + String appHome = System.getProperty(CoreOptions.APP_HOME.key()); if (appHome == null) { throw new IllegalArgumentException( - String.format("%s is not found on System env.", ConfigKeys.KEY_APP_HOME())); + String.format("%s is not found on System env.", CoreOptions.APP_HOME.key())); } File libPath = new File(appHome + "/lib"); @@ -209,21 +197,15 @@ private static void addShimsUrls(FlinkVersion flinkVersion, Consumer addSh LOG.logInfo(includeReason + jarName); } } - // shims-base-v2 must precede shims-base: it redeclares a subset of its classes for Flink - // 2.x, and a URL classloader takes the first match. Directory listing order is arbitrary, - // so leaving this to chance means the Flink version a class was compiled for is decided by - // the filesystem. - matched.sort( - Comparator.comparing(file -> file.getName().startsWith(FLINK_SHIMS_BASE_V2_PREFIX) ? 0 : 1)); matched.forEach(addShimUrl); } - private static String matchShimIncludeReason( - String jarName, String majorVersion, String scalaVersion) { + static String matchShimIncludeReason( + String jarName, String majorVersion, String scalaVersion) { if (jarName.startsWith(FLINK_SHIMS_PREFIX)) { // The shims artifacts carried a _${scala.binary.version} suffix until they were renamed // without it; both spellings are accepted so the jar is matched either way. Getting this - // wrong is silent — a non-matching jar is simply left out of the classloader, and the + // wrong is silent: a non-matching jar is simply left out of the classloader, and the // job fails much later with a NoClassDefFoundError for a class the shims needed. String prefixVer = FLINK_SHIMS_PREFIX + "-" + majorVersion; return jarName.startsWith(prefixVer + "_" + scalaVersion) || jarName.startsWith(prefixVer + "-") @@ -231,13 +213,10 @@ private static String matchShimIncludeReason( : null; } if (jarName.startsWith(FLINK_SHIMS_BASE_PREFIX)) { - // shims-base-v2 redeclares a subset of shims-base for Flink 2.x and inherits the rest, - // so a 2.x target needs both jars with v2 taking precedence (see addShimsUrls), while a - // 1.x target must not see v2 at all — the two share class names, and v2's copies are - // compiled against APIs that moved in 2.x. - return !jarName.startsWith(FLINK_SHIMS_BASE_V2_PREFIX) || isFlink2(majorVersion) - ? "Include flink shims base jar lib: " - : null; + return jarName.startsWith(FLINK_SHIMS_BASE_PREFIX + "-") + && !jarName.startsWith(FLINK_SHIMS_BASE_PREFIX + "-v2-") + ? "Include flink shims base jar lib: " + : null; } if (INCLUDE_PATTERN.matcher(jarName).matches()) { return "Include jar lib: "; @@ -260,10 +239,7 @@ private static ClassLoader getFlinkShimsClassLoader(FlinkVersion flinkVersion) { return SHIMS_CLASS_LOADER_CACHE.computeIfAbsent( flinkVersion.fullVersion(), key -> { - Predicate filter = - file -> !file.getName().startsWith("log4j") && file.getName().endsWith(".jar"); - List libURL = getFlinkHomeLib(flinkVersion.flinkHome, "lib", filter); - List shimsUrls = new ArrayList<>(libURL); + List shimsUrls = getFlinkUrls(flinkVersion, false); addShimsUrls( flinkVersion, @@ -273,14 +249,40 @@ private static ClassLoader getFlinkShimsClassLoader(FlinkVersion flinkVersion) { } }); - return new ChildFirstClassLoader( - shimsUrls.toArray(new URL[0]), - Thread.currentThread().getContextClassLoader(), - PARENT_FIRST_PATTERNS, - jarName -> loadJarFilter(jarName, flinkVersion)); + return createClassLoader(flinkVersion, shimsUrls); }); } + /** + * Collects the target Flink runtime before StreamPark integration artifacts are appended. + * + *

The complete {@code lib} directory is an indivisible compatibility boundary. Selecting + * only table jars would allow their transitive Flink classes to fall back to the Console's + * compile-time version. + */ + private static List getFlinkUrls(FlinkVersion flinkVersion, boolean includeTableOpt) { + Predicate flinkLib = + file -> !file.getName().startsWith("log4j") && file.getName().endsWith(".jar"); + List urls = new ArrayList<>( + getFlinkHomeLib(flinkVersion.flinkHome, "lib", flinkLib)); + if (includeTableOpt) { + Predicate flinkTable = + file -> file.getName().startsWith("flink-table") && file.getName().endsWith(".jar"); + urls.addAll(getFlinkHomeLib(flinkVersion.flinkHome, "opt", flinkTable)); + } + return urls; + } + + private static ClassLoader createClassLoader( + FlinkVersion flinkVersion, + List urls) { + return new ChildFirstClassLoader( + urls.toArray(new URL[0]), + Thread.currentThread().getContextClassLoader(), + PARENT_FIRST_PATTERNS, + jarName -> loadJarFilter(jarName, flinkVersion)); + } + private static List getFlinkHomeLib( String flinkHome, String childDir, diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/ExitSecurityManagerTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/FlinkClientTest.java similarity index 82% rename from streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/ExitSecurityManagerTest.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/FlinkClientTest.java index b55d5ca1ba..0277416596 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/ExitSecurityManagerTest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/FlinkClientTest.java @@ -15,20 +15,18 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.test; - -import org.apache.streampark.flink.client.ExitSecurityManager; +package org.apache.streampark.flink.client; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; -class ExitSecurityManagerTest { +class FlinkClientTest { @Test void checkExitShouldThrowSecurityException() { - ExitSecurityManager securityManager = new ExitSecurityManager(); + FlinkClient.ExitSecurityManager securityManager = new FlinkClient.ExitSecurityManager(); assertThatThrownBy(() -> securityManager.checkExit(0)) .isInstanceOf(SecurityException.class) .hasMessageContaining("System.exit(0)"); @@ -36,7 +34,7 @@ void checkExitShouldThrowSecurityException() { @Test void checkPermissionShouldAllowAllPermissions() { - ExitSecurityManager securityManager = new ExitSecurityManager(); + FlinkClient.ExitSecurityManager securityManager = new FlinkClient.ExitSecurityManager(); assertThatCode(() -> securityManager.checkPermission(null)).doesNotThrowAnyException(); } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/bean/ClientBeanUtilsTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/request/ClientRequestUtilsTest.java similarity index 71% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/bean/ClientBeanUtilsTest.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/request/ClientRequestUtilsTest.java index af4b5c1fd5..36bff38ae1 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/bean/ClientBeanUtilsTest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/request/ClientRequestUtilsTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.request; import org.junit.jupiter.api.Test; @@ -25,27 +25,23 @@ import static org.assertj.core.api.Assertions.assertThat; -class ClientBeanUtilsTest { +class ClientRequestUtilsTest { @Test - void toSerializableMapShouldConvertNonSerializableValues() { + void convertNonSerializableValues() { Map input = new HashMap<>(); input.put("serializable", "value"); input.put("custom", new Object()); - Map result = ClientBeanUtils.toSerializableMap(input); + Map result = ClientRequestUtils.toSerializableMap(input); assertThat(result).containsEntry("serializable", "value"); assertThat(result.get("custom")).isNotNull(); } @Test - void copyPropertiesMapShouldReturnNullForNullInput() { - assertThat(ClientBeanUtils.copyPropertiesMap(null)).isNull(); - } - - @Test - void toSerializableMapShouldReturnNullForNullInput() { - assertThat(ClientBeanUtils.toSerializableMap(null)).isNull(); + void returnEmptyMapsForNull() { + assertThat(ClientRequestUtils.copyPropertiesMap(null)).isEmpty(); + assertThat(ClientRequestUtils.toSerializableMap(null)).isEmpty(); } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java new file mode 100644 index 0000000000..e352827cfd --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.proxy; + +import org.apache.streampark.common.configuration.option.CoreOptions; +import org.apache.streampark.common.core.FlinkVersion; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; + +class FlinkShimsProxyTest { + + private static final String VERSION = "3.0.0.jar"; + + @TempDir + Path temporaryDirectory; + + @Test + void includeOnlyTheSharedBaseArtifact() { + assertThat(includeReason("streampark-flink-shims-base-" + VERSION, "1.20")).isNotNull(); + assertThat(includeReason("streampark-flink-shims-base-" + VERSION, "2.0")).isNotNull(); + assertThat(includeReason("streampark-flink-shims-base-v2-" + VERSION, "1.20")).isNull(); + assertThat(includeReason("streampark-flink-shims-base-v2-" + VERSION, "2.0")).isNull(); + } + + @Test + void includeOnlyRequestedShim() { + assertThat(includeReason("streampark-flink-shims_flink-2.0-" + VERSION, "2.0")).isNotNull(); + assertThat(includeReason("streampark-flink-shims_flink-2.1-" + VERSION, "2.0")).isNull(); + assertThat(includeReason("streampark-flink-shims_flink-1.20_2.12-" + VERSION, "1.20")) + .isNotNull(); + } + + @Test + void sqlLoaderIncludesTargetDist() throws Exception { + Path flinkHome = Files.createDirectories(temporaryDirectory.resolve("flink")); + Path flinkLib = Files.createDirectories(flinkHome.resolve("lib")); + Path flinkOpt = Files.createDirectories(flinkHome.resolve("opt")); + Files.createFile(flinkLib.resolve("flink-dist-1.19.99.jar")); + Files.createFile(flinkLib.resolve("flink-table-api-1.19.99.jar")); + Files.createFile(flinkOpt.resolve("flink-table-planner-1.19.99.jar")); + Path appHome = Files.createDirectories(temporaryDirectory.resolve("streampark")); + Files.createDirectories(appHome.resolve("lib")); + + String propertyKey = CoreOptions.APP_HOME.key(); + String previousAppHome = System.getProperty(propertyKey); + try { + System.setProperty(propertyKey, appHome.toString()); + URL[] urls = + FlinkShimsProxy.proxyVerifySql( + new FlinkVersion(flinkHome.toString()), + loader -> ((URLClassLoader) loader).getURLs()); + + assertThat(Arrays.stream(urls).map(url -> Path.of(url.getPath()).getFileName().toString())) + .contains( + "flink-dist-1.19.99.jar", + "flink-table-api-1.19.99.jar", + "flink-table-planner-1.19.99.jar"); + } finally { + if (previousAppHome == null) { + System.clearProperty(propertyKey); + } else { + System.setProperty(propertyKey, previousAppHome); + } + } + } + + private static String includeReason(String jarName, String majorVersion) { + return FlinkShimsProxy.matchShimIncludeReason(jarName, majorVersion, "2.12"); + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/FlinkClientEntrypoint.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/FlinkClientEntrypoint.java index 9a075434f6..5fd22071fe 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/FlinkClientEntrypoint.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/FlinkClientEntrypoint.java @@ -18,16 +18,7 @@ package org.apache.streampark.flink.client; import org.apache.streampark.common.enums.FlinkDeployMode; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.DeployRequest; -import org.apache.streampark.flink.client.bean.DeployResponse; -import org.apache.streampark.flink.client.bean.SavepointResponse; -import org.apache.streampark.flink.client.bean.ShutDownRequest; -import org.apache.streampark.flink.client.bean.ShutDownResponse; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; +import org.apache.streampark.flink.client.impl.AbstractFlinkClient; import org.apache.streampark.flink.client.impl.KubernetesNativeApplicationClient; import org.apache.streampark.flink.client.impl.KubernetesNativeSessionClient; import org.apache.streampark.flink.client.impl.LocalClient; @@ -35,7 +26,16 @@ import org.apache.streampark.flink.client.impl.YarnApplicationClient; import org.apache.streampark.flink.client.impl.YarnPerJobClient; import org.apache.streampark.flink.client.impl.YarnSessionClient; -import org.apache.streampark.flink.client.trait.FlinkClientTrait; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.DeployRequest; +import org.apache.streampark.flink.client.request.ShutdownRequest; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.DeployResponse; +import org.apache.streampark.flink.client.response.SavepointResponse; +import org.apache.streampark.flink.client.response.ShutdownResponse; +import org.apache.streampark.flink.client.response.SubmitResponse; import org.apache.flink.util.FlinkException; @@ -45,7 +45,8 @@ /** Entry point for Flink client operations routed by deploy mode. */ public final class FlinkClientEntrypoint { - private static final Map CLIENTS = new EnumMap<>(FlinkDeployMode.class); + private static final Map CLIENTS = + new EnumMap<>(FlinkDeployMode.class); static { CLIENTS.put(FlinkDeployMode.LOCAL, LocalClient.INSTANCE); @@ -66,30 +67,33 @@ private FlinkClientEntrypoint() { @FunctionalInterface private interface ClientInvoker { - R invoke(FlinkClientTrait client) throws FlinkException; + R invoke(AbstractFlinkClient client) throws FlinkException; } private static R invokeClient( FlinkDeployMode deployMode, ClientInvoker invoker, String action) throws FlinkException { - FlinkClientTrait client = CLIENTS.get(deployMode); + AbstractFlinkClient client = CLIENTS.get(deployMode); if (client != null) { return invoker.invoke(client); } throw new UnsupportedOperationException("Unsupported " + deployMode + " " + action); } + /** Submits an application through the client selected by its deploy mode. */ public static SubmitResponse submit(SubmitRequest submitRequest) throws FlinkException { return invokeClient( submitRequest.deployMode(), client -> client.submit(submitRequest), "submit"); } + /** Cancels a running job through the client selected by its deploy mode. */ public static CancelResponse cancel(CancelRequest cancelRequest) throws FlinkException { return invokeClient( cancelRequest.deployMode(), client -> client.cancel(cancelRequest), "cancel"); } + /** Triggers a savepoint through the client selected by its deploy mode. */ public static SavepointResponse triggerSavepoint(TriggerSavepointRequest savepointRequest) throws FlinkException { return invokeClient( savepointRequest.deployMode(), @@ -97,6 +101,7 @@ public static SavepointResponse triggerSavepoint(TriggerSavepointRequest savepoi "triggerSavepoint"); } + /** Deploys a persistent session cluster. */ public static DeployResponse deploy(DeployRequest deployRequest) throws Exception { if (deployRequest.deployMode() == FlinkDeployMode.YARN_SESSION) { return YarnSessionClient.INSTANCE.deploy(deployRequest); @@ -108,14 +113,15 @@ public static DeployResponse deploy(DeployRequest deployRequest) throws Exceptio "Unsupported " + deployRequest.deployMode() + " deploy cluster "); } - public static ShutDownResponse shutdown(ShutDownRequest shutDownRequest) throws Exception { - if (shutDownRequest.deployMode() == FlinkDeployMode.YARN_SESSION) { - return YarnSessionClient.INSTANCE.shutdown(shutDownRequest); + /** Shuts down a persistent session cluster. */ + public static ShutdownResponse shutdown(ShutdownRequest shutdownRequest) throws Exception { + if (shutdownRequest.deployMode() == FlinkDeployMode.YARN_SESSION) { + return YarnSessionClient.INSTANCE.shutdown(shutdownRequest); } - if (shutDownRequest.deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_SESSION) { - return KubernetesNativeSessionClient.INSTANCE.shutdown(shutDownRequest); + if (shutdownRequest.deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_SESSION) { + return KubernetesNativeSessionClient.INSTANCE.shutdown(shutdownRequest); } throw new UnsupportedOperationException( - "Unsupported " + shutDownRequest.deployMode() + " shutdown cluster "); + "Unsupported " + shutdownRequest.deployMode() + " shutdown cluster "); } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/FlinkJobGraphBuilder.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/FlinkJobGraphBuilder.java new file mode 100644 index 0000000000..794a240375 --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/FlinkJobGraphBuilder.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.bean; + +import org.apache.streampark.common.enums.FlinkJobType; +import org.apache.streampark.flink.client.request.SubmitRequest; + +import org.apache.flink.client.deployment.application.ApplicationConfiguration; +import org.apache.flink.client.program.PackagedProgram; +import org.apache.flink.client.program.PackagedProgramUtils; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.jobgraph.JobGraph; + +import java.io.File; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Builds a JobGraph with the classloader of the registered Flink version. */ +public final class FlinkJobGraphBuilder { + + private FlinkJobGraphBuilder() { + } + + /** Creates the packaged program and its executable job graph. */ + public static Result build( + Configuration configuration, + SubmitRequest request, + File jarFile) throws Exception { + PackagedProgram.Builder builder = + PackagedProgram.newBuilder() + .setSavepointRestoreSettings( + SubmitRequestResolver.savepointRestoreSettings(request)) + .setEntryPointClassName( + configuration + .getOptional(ApplicationConfiguration.APPLICATION_MAIN_CLASS) + .orElseThrow( + () -> new IllegalStateException( + "Application main class is not configured"))) + .setArguments( + configuration + .getOptional(ApplicationConfiguration.APPLICATION_ARGS) + .orElse(Collections.emptyList()) + .toArray(new String[0])); + + if (request.jobType() != FlinkJobType.PYFLINK) { + builder.setJarFile(jarFile); + boolean flinkSqlJob = request.jobType() == FlinkJobType.FLINK_SQL; + builder.setUserClassPaths(userClassPaths(request, !flinkSqlJob)); + builder.setConfiguration(parentFirstConfiguration()); + } + + ClassLoader shimsClassLoader = Thread.currentThread().getContextClassLoader(); + ClassLoader programParent = + request.jobType() == FlinkJobType.PYFLINK + ? shimsClassLoader + : programParentClassLoader(shimsClassLoader); + + Thread.currentThread().setContextClassLoader(programParent); + + try { + PackagedProgram program = builder.build(); + try { + Configuration jobGraphConfiguration = new Configuration(configuration); + if (request.jobType() != FlinkJobType.PYFLINK) { + jobGraphConfiguration.addAll(parentFirstConfiguration()); + } + JobGraph jobGraph = + PackagedProgramUtils.createJobGraph( + program, + jobGraphConfiguration, + SubmissionConfigurationBuilder.parallelism(request), + null, + false); + return new Result(program, jobGraph); + } catch (Exception failure) { + closeAfterBuildFailure(request, program, failure); + throw failure; + } + } finally { + Thread.currentThread().setContextClassLoader(shimsClassLoader); + } + } + + private static void closeAfterBuildFailure( + SubmitRequest request, + PackagedProgram program, + Exception failure) { + if (!SubmitRequestResolver.canSafelyClosePackagedProgram(request)) { + return; + } + try { + program.close(); + } catch (Exception closeFailure) { + failure.addSuppressed(closeFailure); + } + } + + private static Configuration parentFirstConfiguration() { + Configuration configuration = new Configuration(); + configuration.setString( + "classloader.parent-first-patterns.additional", + "org.apache.streampark.;org.apache.flink.;org.yaml."); + return configuration; + } + + /** + * Creates the parent used by Flink's user-code classloader. + * + *

StreamPark and Flink classes come from the version-isolated shims classloader. SnakeYAML + * comes from the console classloader to avoid conflicts with SQL job fat jars. + */ + private static ClassLoader programParentClassLoader(ClassLoader shimsClassLoader) { + ClassLoader consoleClassLoader = FlinkJobGraphBuilder.class.getClassLoader(); + return new ClassLoader(shimsClassLoader) { + + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + synchronized (getClassLoadingLock(name)) { + Class loaded = findLoadedClass(name); + if (loaded == null) { + if (name.startsWith("org.apache.streampark.") + || name.startsWith("org.apache.flink.")) { + loaded = getParent().loadClass(name); + } else if (name.startsWith("org.yaml.")) { + loaded = consoleClassLoader.loadClass(name); + } + } + if (loaded == null) { + loaded = super.loadClass(name, resolve); + } else if (resolve) { + resolveClass(loaded); + } + return loaded; + } + } + }; + } + + /** + * Resolves the classpath used while building the JobGraph. + * + *

Thin uploaded jars need {@code flink-dist} on Flink 1.20 and later. SQL fat jars omit it + * and use parent-first delegation to prevent dependency conflicts. + */ + private static List userClassPaths(SubmitRequest request, boolean includeFlinkDist) { + List classPaths = new ArrayList<>(SubmitRequestResolver.classPaths(request)); + classPaths.removeIf( + url -> { + String path = url.getPath(); + return path.contains("flink-shaded-force-shading") + || (!includeFlinkDist && path.contains("flink-dist")); + }); + return classPaths; + } + + /** Result containing resources created while building a JobGraph. */ + public static final class Result { + + private final PackagedProgram program; + private final JobGraph jobGraph; + + private Result(PackagedProgram program, JobGraph jobGraph) { + this.program = program; + this.jobGraph = jobGraph; + } + + public PackagedProgram program() { + return program; + } + + public JobGraph jobGraph() { + return jobGraph; + } + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ClientBeanUtils.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/RemoteWorkspace.java similarity index 52% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ClientBeanUtils.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/RemoteWorkspace.java index e8ad5eb252..38c013f76a 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/bean/ClientBeanUtils.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/RemoteWorkspace.java @@ -17,28 +17,26 @@ package org.apache.streampark.flink.client.bean; -import org.apache.streampark.common.conf.FlinkVersion; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.Workspace; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.flink.util.FlinkUtils; import org.apache.commons.io.FileUtils; -import javax.annotation.Nullable; - import java.io.File; import java.io.IOException; -import java.io.Serializable; import java.io.UncheckedIOException; -import java.util.HashMap; -import java.util.Map; -final class ClientBeanUtils { +/** Resolved HDFS paths used while submitting a Flink job through YARN. */ +public class RemoteWorkspace { - private ClientBeanUtils() { - } + private final String flinkDistJar; + private final String flinkLib; + private final String flinkPlugins; + private final String appJars; - static HdfsWorkspace createHdfsWorkspace(FlinkVersion flinkVersion) { - Workspace workspace = Workspace.remote(); + public static RemoteWorkspace resolve(FlinkVersion flinkVersion) { + Workspace workspace = Workspace.REMOTE; String flinkHome = flinkVersion.flinkHome; File flinkHomeDir = new File(flinkHome); String flinkName; @@ -51,40 +49,38 @@ static HdfsWorkspace createHdfsWorkspace(FlinkVersion flinkVersion) { throw new UncheckedIOException( "Failed to resolve Flink home directory name: " + flinkHome, e); } - String flinkHdfsHome = workspace.APP_FLINK() + "/" + flinkName; - return new HdfsWorkspace( - flinkName, - flinkHome, + String flinkHdfsHome = workspace.flink + "/" + flinkName; + return new RemoteWorkspace( FlinkUtils.getFlinkDistJar(flinkHome), flinkHdfsHome + "/lib", flinkHdfsHome + "/plugins", - workspace.APP_JARS()); + workspace.jars); } - @Nullable - static Map copyPropertiesMap(@Nullable Map properties) { - if (properties == null) { - return null; - } - Map result = new HashMap<>(); - result.putAll(properties); - return result; + private RemoteWorkspace( + String flinkDistJar, + String flinkLib, + String flinkPlugins, + String appJars) { + this.flinkDistJar = flinkDistJar; + this.flinkLib = flinkLib; + this.flinkPlugins = flinkPlugins; + this.appJars = appJars; } - @Nullable - static Map toSerializableMap(@Nullable Map properties) { - if (properties == null) { - return null; - } - Map result = new HashMap<>(); - for (Map.Entry entry : properties.entrySet()) { - Object value = entry.getValue(); - if (value instanceof Serializable) { - result.put(entry.getKey(), (Serializable) value); - } else if (value != null) { - result.put(entry.getKey(), value.toString()); - } - } - return result; + public String flinkDistJar() { + return flinkDistJar; + } + + public String flinkLib() { + return flinkLib; + } + + public String flinkPlugins() { + return flinkPlugins; + } + + public String appJars() { + return appJars; } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SessionClusterRestClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SessionClusterRestClient.java new file mode 100644 index 0000000000..6cf09638a9 --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SessionClusterRestClient.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.bean; + +import org.apache.streampark.common.util.AssertUtils; +import org.apache.streampark.common.util.JsonUtils; +import org.apache.streampark.flink.client.configuration.FlinkSavepointOptions; + +import org.apache.streampark.shaded.com.fasterxml.jackson.databind.JsonNode; + +import org.apache.flink.client.deployment.application.ApplicationConfiguration; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.CoreOptions; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.io.UncheckedIOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.security.AccessController; +import java.security.PrivilegedExceptionAction; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** Submits Flink jobs to session clusters through the Flink REST API. */ +public final class SessionClusterRestClient { + + private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(30); + private static final Duration REQUEST_TIMEOUT = Duration.ofMinutes(5); + + private SessionClusterRestClient() { + } + + /** Uploads a job JAR and starts it on the target session cluster. */ + public static String submit( + String jobManagerUrl, + File jobJar, + Configuration configuration) throws Exception { + AssertUtils.required(jobJar.isFile(), "Flink job JAR does not exist: " + jobJar); + HttpClient httpClient = + HttpClient.newBuilder().connectTimeout(CONNECT_TIMEOUT).build(); + + String boundary = "----StreamParkBoundary" + System.currentTimeMillis(); + HttpRequest uploadRequest = + HttpRequest.newBuilder() + .uri(endpoint(jobManagerUrl, "/jars/upload")) + .timeout(REQUEST_TIMEOUT) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(multipartBody(boundary, jobJar)) + .build(); + HttpResponse uploadResponse = send(httpClient, uploadRequest); + requireSuccess("upload job JAR", uploadResponse); + JarUploadResponse upload = parseUploadResponse(uploadResponse.body()); + + HttpRequest runRequest = + HttpRequest.newBuilder() + .uri(endpoint(jobManagerUrl, "/jars/" + upload.jarId() + "/run")) + .timeout(REQUEST_TIMEOUT) + .header("Content-Type", "application/json") + .POST( + HttpRequest.BodyPublishers.ofString( + JsonUtils.write(new JarRunRequest(configuration)))) + .build(); + HttpResponse runResponse = send(httpClient, runRequest); + requireSuccess("start uploaded job", runResponse); + return parseJobId(runResponse.body()); + } + + private static URI endpoint(String jobManagerUrl, String path) { + String normalized = + jobManagerUrl.endsWith("/") + ? jobManagerUrl.substring(0, jobManagerUrl.length() - 1) + : jobManagerUrl; + return URI.create(normalized + path); + } + + private static HttpResponse send(HttpClient client, HttpRequest request) throws Exception { + return AccessController.doPrivileged( + (PrivilegedExceptionAction>) () -> client.send( + request, + HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8))); + } + + private static void requireSuccess(String action, HttpResponse response) { + int status = response.statusCode(); + if (status < 200 || status >= 300) { + throw new IllegalStateException( + "Failed to " + action + ": HTTP " + status + ", response=" + response.body()); + } + } + + private static HttpRequest.BodyPublisher multipartBody(String boundary, File jobJar) { + byte[] header = + ("--" + + boundary + + "\r\nContent-Disposition: form-data; name=\"jarfile\"; filename=\"" + + jobJar.getName() + + "\"\r\nContent-Type: application/java-archive\r\n\r\n") + .getBytes(StandardCharsets.UTF_8); + byte[] footer = + ("\r\n--" + boundary + "--\r\n").getBytes(StandardCharsets.UTF_8); + return HttpRequest.BodyPublishers.ofInputStream( + () -> multipartStream(header, jobJar, footer)); + } + + private static InputStream multipartStream(byte[] header, File jobJar, byte[] footer) { + try { + List streams = + Arrays.asList( + new ByteArrayInputStream(header), + new FileInputStream(jobJar), + new ByteArrayInputStream(footer)); + return new SequenceInputStream(Collections.enumeration(streams)); + } catch (IOException e) { + throw new UncheckedIOException("Failed to open Flink job JAR: " + jobJar, e); + } + } + + private static JarUploadResponse parseUploadResponse(String responseBody) { + try { + JsonNode node = JsonUtils.read(responseBody, JsonNode.class); + JarUploadResponse response = + new JarUploadResponse( + node.has("filename") ? node.get("filename").asText(null) : null, + node.has("status") ? node.get("status").asText(null) : null); + AssertUtils.required( + response.isSuccessful(), + "Flink rejected the uploaded job JAR: " + responseBody); + return response; + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException( + "Invalid response from Flink JAR upload endpoint: " + responseBody, e); + } + } + + private static String parseJobId(String responseBody) { + try { + JsonNode node = JsonUtils.read(responseBody, JsonNode.class); + String jobId = null; + if (node.has("jobid")) { + jobId = node.get("jobid").asText(null); + } else if (node.has("jobId")) { + jobId = node.get("jobId").asText(null); + } + AssertUtils.required( + jobId != null && !jobId.isBlank(), + "Flink did not return a job ID: " + responseBody); + return jobId; + } catch (RuntimeException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException( + "Invalid response from Flink JAR run endpoint: " + responseBody, e); + } + } + + private static final class JarUploadResponse { + + private final String filename; + private final String status; + + private JarUploadResponse(String filename, String status) { + this.filename = filename; + this.status = status; + } + + private boolean isSuccessful() { + return filename != null + && !filename.isBlank() + && "success".equalsIgnoreCase(status); + } + + private String jarId() { + return filename.substring(filename.lastIndexOf('/') + 1); + } + } + + private static final class JarRunRequest { + + private final String entryClass; + private final String programArgs; + private final String parallelism; + private final String savepointPath; + private final boolean allowNonRestoredState; + + private JarRunRequest(Configuration configuration) { + this.entryClass = + configuration.get(ApplicationConfiguration.APPLICATION_MAIN_CLASS); + List args = + configuration.get(ApplicationConfiguration.APPLICATION_ARGS); + this.programArgs = args == null ? null : String.join(" ", args); + this.parallelism = + String.valueOf(configuration.get(CoreOptions.DEFAULT_PARALLELISM)); + this.savepointPath = + configuration.get(FlinkSavepointOptions.SAVEPOINT_PATH); + this.allowNonRestoredState = + configuration.get( + FlinkSavepointOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE); + } + + public String getEntryClass() { + return entryClass; + } + + public String getProgramArgs() { + return programArgs; + } + + public String getParallelism() { + return parallelism; + } + + public String getSavepointPath() { + return savepointPath; + } + + public boolean isAllowNonRestoredState() { + return allowNonRestoredState; + } + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmissionConfigurationBuilder.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmissionConfigurationBuilder.java new file mode 100644 index 0000000000..e425792089 --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmissionConfigurationBuilder.java @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.bean; + +import org.apache.streampark.common.configuration.CommandLineParser; +import org.apache.streampark.common.configuration.CommandLineTokenizer; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.FlinkRunOption; +import org.apache.streampark.common.configuration.Workspace; +import org.apache.streampark.common.configuration.option.ApplicationOptions; +import org.apache.streampark.common.enums.ApplicationType; +import org.apache.streampark.common.enums.FlinkDeployMode; +import org.apache.streampark.common.enums.FlinkJobType; +import org.apache.streampark.common.enums.FlinkRestoreMode; +import org.apache.streampark.common.fs.FsOperator; +import org.apache.streampark.common.util.AssertUtils; +import org.apache.streampark.common.util.ClassLoaderUtils; +import org.apache.streampark.common.util.DeflaterUtils; +import org.apache.streampark.common.util.FlinkConfigurationUtils; +import org.apache.streampark.common.util.LoggerSupport; +import org.apache.streampark.common.util.SystemPropertyUtils; +import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.configuration.FlinkSavepointOptions; +import org.apache.streampark.flink.client.request.SubmitRequest; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.client.cli.CliFrontend; +import org.apache.flink.client.cli.CliFrontendParser; +import org.apache.flink.client.cli.CustomCommandLine; +import org.apache.flink.client.cli.ExecutionConfigAccessor; +import org.apache.flink.client.cli.ProgramOptions; +import org.apache.flink.client.deployment.application.ApplicationConfiguration; +import org.apache.flink.client.program.PackagedProgramUtils; +import org.apache.flink.configuration.CheckpointingOptions; +import org.apache.flink.configuration.ConfigConstants; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.CoreOptions; +import org.apache.flink.configuration.DeploymentOptions; +import org.apache.flink.configuration.ExecutionOptions; +import org.apache.flink.configuration.PipelineOptions; +import org.apache.flink.configuration.PipelineOptionsInternal; +import org.apache.flink.python.PythonOptions; +import org.apache.flink.util.Preconditions; + +import java.io.File; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** Builds the effective Flink configuration for a submission request. */ +public final class SubmissionConfigurationBuilder extends LoggerSupport { + + private static final SubmissionConfigurationBuilder INSTANCE = + new SubmissionConfigurationBuilder(); + + private static final String PARAM_KEY_FLINK_CONF = + CommandLineParser.LONG_OPTION_PREFIX + FlinkOptions.FLINK_CONFIGURATION.key(); + private static final String PARAM_KEY_FLINK_CONF_STANDARD_YAML = + CommandLineParser.LONG_OPTION_PREFIX + + FlinkOptions.FLINK_CONFIGURATION_STANDARD_YAML.key(); + private static final String PARAM_KEY_FLINK_SQL = + CommandLineParser.LONG_OPTION_PREFIX + ApplicationOptions.SQL.key(); + private static final String PARAM_KEY_APP_CONF = + CommandLineParser.LONG_OPTION_PREFIX + ApplicationOptions.CONFIG.key(); + private static final String PARAM_KEY_APP_NAME = + CommandLineParser.LONG_OPTION_PREFIX + ApplicationOptions.NAME.key(); + private static final String PARAM_KEY_FLINK_PARALLELISM = + CommandLineParser.LONG_OPTION_PREFIX + FlinkOptions.PARALLELISM.key(); + + private SubmissionConfigurationBuilder() { + } + + /** Builds the configuration used by a deployment client. */ + public static Configuration build(SubmitRequest request) throws Exception { + return INSTANCE.buildConfiguration(request); + } + + /** Extracts a Flink configuration from dynamic cluster properties. */ + public static Configuration extract(String flinkHome, Map properties) throws Exception { + return INSTANCE.extractConfiguration(flinkHome, properties); + } + + /** Loads the registered Flink installation's default configuration. */ + public static Configuration loadDefault(String flinkHome) { + try { + return Configuration.fromMap(FlinkConfigurationUtils.loadFlinkHome(flinkHome)); + } catch (Exception ignored) { + return new Configuration(); + } + } + + /** Resolves the effective parallelism for the request. */ + public static int parallelism(SubmitRequest request) { + Object configured = request.properties().get(FlinkOptions.PARALLELISM.key()); + if (configured != null) { + return Integer.parseInt(configured.toString()); + } + return loadDefault(request.flinkVersion().getFlinkHome()) + .get(CoreOptions.DEFAULT_PARALLELISM, CoreOptions.DEFAULT_PARALLELISM.defaultValue()); + } + + /** Reads an option from the registered Flink installation. */ + public static T getDefaultOption(String flinkHome, ConfigOption option) { + return loadDefault(flinkHome).get(option); + } + + private Configuration buildConfiguration(SubmitRequest request) throws Exception { + CommandLineAndConfiguration cli = parseCommandLine(request); + Configuration configuration = cli.configuration; + applyJobTypeConfiguration(request, cli.commandLine, configuration); + applyPipelineConfiguration(request, configuration); + applyCheckpointDefaults(request, configuration); + applySavepointConfiguration(request, configuration); + applyEnvironmentProperties(request, configuration); + return configuration; + } + + private void applyJobTypeConfiguration( + SubmitRequest request, + CommandLine commandLine, + Configuration configuration) throws Exception { + if (request.jobType() == FlinkJobType.PYFLINK) { + applyPyFlinkConfiguration(request, configuration); + return; + } + + File userJar = SubmitRequestResolver.userJarFile(request); + if (userJar == null) { + return; + } + URI jarUri = PackagedProgramUtils.resolveURI(userJar.getAbsolutePath()); + ProgramOptions programOptions = ProgramOptions.create(commandLine); + ExecutionConfigAccessor.fromProgramOptions( + programOptions, Collections.singletonList(jarUri.toString())) + .applyToConfiguration(configuration); + } + + private void applyPyFlinkConfiguration( + SubmitRequest request, Configuration configuration) throws Exception { + String pythonVenv = Workspace.LOCAL.pythonVenv; + AssertUtils.required( + FsOperator.lfs().exists(pythonVenv), pythonVenv + " file does not exist"); + + FlinkConfigurationOps.setIfPresent( + configuration, PythonOptions.PYTHON_ARCHIVES, pythonVenv); + FlinkConfigurationOps.setIfPresent( + configuration, PythonOptions.PYTHON_CLIENT_EXECUTABLE, Constants.PYTHON_EXECUTABLE); + FlinkConfigurationOps.setIfPresent( + configuration, PythonOptions.PYTHON_EXECUTABLE, Constants.PYTHON_EXECUTABLE); + + if (StringUtils.isBlank(System.getenv(ConfigConstants.ENV_FLINK_OPT_DIR))) { + String flinkOptPath = request.flinkVersion().getFlinkHome() + "/opt"; + SystemPropertyUtils.setEnv(ConfigConstants.ENV_FLINK_OPT_DIR, flinkOptPath); + logInfo( + "Set temporary environment variable " + + ConfigConstants.ENV_FLINK_OPT_DIR + + "=" + + flinkOptPath); + } + } + + private void applyPipelineConfiguration( + SubmitRequest request, Configuration configuration) { + FlinkConfigurationOps.setIfPresent( + configuration, PipelineOptions.NAME, SubmitRequestResolver.effectiveApplicationName(request)); + FlinkConfigurationOps.setIfPresent( + configuration, DeploymentOptions.TARGET, request.deployMode().getName()); + FlinkConfigurationOps.setIfPresent( + configuration, FlinkSavepointOptions.SAVEPOINT_PATH, request.savePoint()); + FlinkConfigurationOps.setIfPresent( + configuration, + ApplicationConfiguration.APPLICATION_MAIN_CLASS, + SubmitRequestResolver.applicationMain(request)); + FlinkConfigurationOps.setIfPresent( + configuration, ApplicationConfiguration.APPLICATION_ARGS, programArguments(request)); + FlinkConfigurationOps.setIfPresent( + configuration, PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, request.jobId()); + } + + private void applyCheckpointDefaults( + SubmitRequest request, Configuration configuration) { + ConfigOption retainedCheckpoints = CheckpointingOptions.MAX_RETAINED_CHECKPOINTS; + if (!request.properties().containsKey(retainedCheckpoints.key())) { + configuration.set( + retainedCheckpoints, + loadDefault(request.flinkVersion().getFlinkHome()).get(retainedCheckpoints)); + } + } + + private void applySavepointConfiguration( + SubmitRequest request, Configuration configuration) { + if (StringUtils.isBlank(request.savePoint())) { + return; + } + FlinkConfigurationOps.setIfPresent( + configuration, FlinkSavepointOptions.SAVEPOINT_PATH, request.savePoint()); + configuration.set( + FlinkSavepointOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE, + SubmitRequestResolver.allowNonRestoredState(request)); + + if (request.restoreMode() != null + && request.flinkVersion().checkVersion(FlinkRestoreMode.SINCE_FLINK_VERSION)) { + configuration.setString(FlinkRestoreMode.RESTORE_MODE, request.restoreMode().getName()); + } + } + + private void applyEnvironmentProperties( + SubmitRequest request, Configuration configuration) { + if (MapUtils.isEmpty(request.properties())) { + return; + } + + Object jvmOptions = request.properties().get(CoreOptions.FLINK_JVM_OPTIONS.key()); + if (jvmOptions != null && !jvmOptions.toString().contains("-Dfile.encoding=")) { + configuration.setString( + CoreOptions.FLINK_JVM_OPTIONS.key(), + "-Dfile.encoding=UTF-8 " + jvmOptions); + } + + request.properties().forEach( + (key, value) -> { + if (key.startsWith("env.")) { + logInfo("Environment option: " + key + "=" + value); + configuration.setString(key, value.toString()); + } + }); + } + + private CommandLineAndConfiguration parseCommandLine(SubmitRequest request) throws Exception { + String flinkHome = request.flinkVersion().getFlinkHome(); + Options options = commandLineOptions(flinkHome); + List arguments = cliArguments(request, buildOptionMap(request, options)); + logInfo("Flink CLI arguments: " + String.join(" ", arguments)); + + CommandLine commandLine = + FlinkRunOption.parse(options, arguments.toArray(new String[0]), true); + CustomCommandLine activeCommandLine = + activeCommandLine(customCommandLines(flinkHome), commandLine); + return new CommandLineAndConfiguration( + commandLine, applyConfiguration(flinkHome, activeCommandLine, commandLine)); + } + + private Map buildOptionMap(SubmitRequest request, Options commandLineOptions) { + Map options = new HashMap<>(); + SubmitRequestResolver.applicationOptions(request).forEach( + (key, value) -> { + if (!commandLineOptions.hasOption(key)) { + logWarn("Ignoring unsupported Flink CLI option: " + key); + return; + } + String option = commandLineOptions.getOption(key.trim()).getOpt(); + Object parsedValue = parseOptionValue(value); + if (Boolean.TRUE.equals(parsedValue)) { + options.put("-" + option, true); + } else if (!(parsedValue instanceof Boolean)) { + options.put("-" + option, parsedValue); + } + }); + + if (request.savePoint() != null) { + options.put("-" + FlinkRunOption.SAVEPOINT_PATH_OPTION.getOpt(), request.savePoint()); + } + options.remove("-e"); + options.remove("--executor"); + options.remove("-t"); + options.remove("--target"); + if (request.deployMode() != null) { + options.put("-t", request.deployMode().getName()); + } + return options; + } + + private List cliArguments(SubmitRequest request, Map options) { + List arguments = new ArrayList<>(); + options.forEach( + (key, value) -> { + arguments.add(key); + if (value instanceof String) { + arguments.add((String) value); + } + }); + request.properties().forEach( + (key, value) -> { + if (!key.startsWith("env.")) { + arguments.add("-D" + key + "=" + value); + } + }); + return arguments; + } + + private List programArguments(SubmitRequest request) { + List arguments = + new ArrayList<>(CommandLineTokenizer.tokenize(request.args())); + if (request.applicationType() == ApplicationType.STREAMPARK_FLINK + || request.jobType() == FlinkJobType.FLINK_SQL) { + addStreamParkArguments(request, arguments); + } + + Object runtimeMode = request.properties().get(ExecutionOptions.RUNTIME_MODE.key()); + if (runtimeMode != null) { + arguments.add("--" + ExecutionOptions.RUNTIME_MODE.key()); + arguments.add(runtimeMode.toString()); + } + + if (request.jobType() == FlinkJobType.PYFLINK + && request.deployMode() != FlinkDeployMode.YARN_APPLICATION) { + File userJar = SubmitRequestResolver.userJarFile(request); + AssertUtils.notNull(userJar); + arguments.add("-py"); + arguments.add(userJar.getAbsolutePath()); + } + return arguments; + } + + private void addStreamParkArguments(SubmitRequest request, List arguments) { + arguments.add(PARAM_KEY_FLINK_CONF); + arguments.add(request.flinkYaml()); + // Carry the persisted syntax snapshot so Flink 1.19 never re-resolves its configuration + // format from installation files that may have changed since registration. + arguments.add(PARAM_KEY_FLINK_CONF_STANDARD_YAML); + arguments.add(Boolean.toString(request.standardYaml())); + arguments.add(PARAM_KEY_APP_NAME); + arguments.add( + DeflaterUtils.zipString(SubmitRequestResolver.effectiveApplicationName(request))); + arguments.add(PARAM_KEY_FLINK_PARALLELISM); + arguments.add(Integer.toString(parallelism(request))); + + if (request.jobType() == FlinkJobType.FLINK_SQL) { + arguments.add(PARAM_KEY_FLINK_SQL); + arguments.add(SubmitRequestResolver.flinkSql(request)); + addApplicationConfiguration(request.appConf(), arguments); + } else if (shouldAddApplicationConfiguration(request.appConf())) { + addApplicationConfiguration(request.appConf(), arguments); + } + } + + private void addApplicationConfiguration(String appConf, List arguments) { + if (appConf != null) { + arguments.add(PARAM_KEY_APP_CONF); + arguments.add(appConf); + } + } + + private Configuration extractConfiguration( + String flinkHome, Map properties) throws Exception { + List arguments = new ArrayList<>(); + if (MapUtils.isNotEmpty(properties)) { + properties.forEach( + (key, value) -> arguments.add("-D" + key + "=" + value.toString().trim())); + } + CommandLine commandLine = + FlinkRunOption.parse( + commandLineOptions(flinkHome), arguments.toArray(new String[0]), true); + CustomCommandLine activeCommandLine = + activeCommandLine(customCommandLines(flinkHome), commandLine); + return applyConfiguration(flinkHome, activeCommandLine, commandLine); + } + + private CustomCommandLine activeCommandLine( + List commandLines, + CommandLine commandLine) { + Preconditions.checkNotNull(commandLine); + for (CustomCommandLine candidate : commandLines) { + if (candidate.isActive(commandLine)) { + return candidate; + } + } + throw new IllegalStateException("No active Flink command line found"); + } + + private List customCommandLines(String flinkHome) { + Configuration defaultConfiguration = loadDefault(flinkHome); + String configurationDirectory = flinkHome + "/conf"; + return ClassLoaderUtils.runAsClassLoader( + SubmissionConfigurationBuilder.class.getClassLoader(), + () -> CliFrontend.loadCustomCommandLines( + defaultConfiguration, configurationDirectory)); + } + + private Options commandLineOptions(String flinkHome) { + Options customOptions = new Options(); + for (CustomCommandLine commandLine : customCommandLines(flinkHome)) { + commandLine.addGeneralOptions(customOptions); + commandLine.addRunOptions(customOptions); + } + return FlinkRunOption.mergeOptions( + CliFrontendParser.getRunCommandOptions(), customOptions); + } + + private Configuration applyConfiguration( + String flinkHome, + CustomCommandLine activeCommandLine, + CommandLine commandLine) throws Exception { + Preconditions.checkNotNull(activeCommandLine, "activeCommandLine must not be null"); + Configuration configuration = new Configuration(); + Configuration defaults = loadDefault(flinkHome); + Set keys = defaults.keySet(); + for (String key : keys) { + String value = defaults.getString(key, null); + if (value != null) { + configuration.setString(key, value); + } + } + configuration.addAll(activeCommandLine.toConfiguration(commandLine)); + return configuration; + } + + private static Object parseOptionValue(String value) { + if ("true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value)) { + return Boolean.parseBoolean(value); + } + return value; + } + + private static boolean shouldAddApplicationConfiguration(String appConf) { + return StringUtils.isNotBlank(appConf) && !appConf.startsWith("json:"); + } + + private static final class CommandLineAndConfiguration { + + private final CommandLine commandLine; + private final Configuration configuration; + + private CommandLineAndConfiguration( + CommandLine commandLine, Configuration configuration) { + this.commandLine = commandLine; + this.configuration = configuration; + } + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequestResolver.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequestResolver.java new file mode 100644 index 0000000000..c85d15e3b3 --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequestResolver.java @@ -0,0 +1,297 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.bean; + +import org.apache.streampark.common.configuration.ConfigurationFormat; +import org.apache.streampark.common.configuration.ConfigurationParser; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.Workspace; +import org.apache.streampark.common.configuration.option.ApplicationOptions; +import org.apache.streampark.common.enums.FlinkDeployMode; +import org.apache.streampark.common.enums.FlinkJobType; +import org.apache.streampark.common.util.AssertUtils; +import org.apache.streampark.common.util.DeflaterUtils; +import org.apache.streampark.common.util.HdfsUtils; +import org.apache.streampark.flink.client.configuration.FlinkSavepointOptions; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.packer.pipeline.BuildResult; +import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse; + +import org.apache.streampark.shaded.com.fasterxml.jackson.core.type.TypeReference; +import org.apache.streampark.shaded.com.fasterxml.jackson.databind.ObjectMapper; + +import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; + +import javax.annotation.Nullable; + +import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** Resolves runtime submission values from the serialized client request. */ +public final class SubmitRequestResolver { + + private static final int CONFIG_SCHEME_LENGTH = 7; + + private SubmitRequestResolver() { + } + + /** Returns application properties without their transport prefix. */ + public static Map applicationProperties(SubmitRequest request) { + return getParameters(request, FlinkOptions.PROPERTY_PREFIX.value()); + } + + /** Returns application options without their transport prefix. */ + public static Map applicationOptions(SubmitRequest request) { + return getParameters(request, FlinkOptions.OPTION_PREFIX.value()); + } + + /** Resolves the application entry class for the submitted job type. */ + @Nullable + public static String applicationMain(SubmitRequest request) { + if (request.jobType() == FlinkJobType.FLINK_SQL) { + return Constants.STREAMPARK_FLINKSQL_CLIENT_CLASS; + } + if (request.jobType() == FlinkJobType.PYFLINK) { + return Constants.PYTHON_FLINK_DRIVER_CLASS_NAME; + } + String mainClass = + applicationProperties(request).get(FlinkOptions.APPLICATION_MAIN_CLASS.key()); + if (mainClass == null && request.appConf() != null) { + mainClass = + loadApplicationConfig(request.appConf()) + .get(FlinkOptions.APPLICATION_MAIN_CLASS.key()); + } + return mainClass; + } + + /** Resolves the explicit or configured application name. */ + @Nullable + public static String effectiveApplicationName(SubmitRequest request) { + return request.appName() == null + ? applicationProperties(request).get(FlinkOptions.PIPELINE_NAME.key()) + : request.appName(); + } + + /** Returns the application-specific libraries available in the local workspace. */ + public static List libraries(SubmitRequest request) { + File libDir = + new File( + new File(Workspace.LOCAL.workspace, String.valueOf(request.id())), "lib"); + File[] files = libDir.listFiles(); + if (files == null) { + return Collections.emptyList(); + } + List urls = new ArrayList<>(files.length); + for (File file : files) { + try { + urls.add(file.toURI().toURL()); + } catch (MalformedURLException e) { + throw new IllegalArgumentException("Invalid library path: " + file, e); + } + } + return urls; + } + + /** Builds the complete user-code classpath for submission. */ + public static List classPaths(SubmitRequest request) { + try { + List classPaths = new ArrayList<>(request.flinkVersion().getFlinkLibs()); + classPaths.addAll(libraries(request)); + return classPaths; + } catch (Exception e) { + throw new IllegalStateException("Failed to resolve the Flink submission classpath", e); + } + } + + /** Returns the SQL payload carried by a Flink SQL request. */ + @Nullable + public static String flinkSql(SubmitRequest request) { + Map extraParameter = request.extraParameter(); + Object sql = extraParameter.get(ApplicationOptions.SQL.key()); + return sql == null ? null : sql.toString(); + } + + /** Returns whether state not represented in a savepoint may be skipped. */ + public static boolean allowNonRestoredState(SubmitRequest request) { + Object value = + request.properties().get(FlinkSavepointOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE.key()); + return value != null && Boolean.parseBoolean(value.toString()); + } + + /** Creates Flink restore settings from the request savepoint options. */ + public static SavepointRestoreSettings savepointRestoreSettings(SubmitRequest request) { + if (request.savePoint() == null || request.savePoint().isEmpty()) { + return SavepointRestoreSettings.none(); + } + return SavepointRestoreSettings.forPath( + request.savePoint(), allowNonRestoredState(request)); + } + + /** Resolves the built user JAR for deployment modes that upload a local artifact. */ + @Nullable + public static File userJarFile(SubmitRequest request) { + if (request.deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) { + return null; + } + validateBuildResult(request); + ShadedBuildResponse buildResult = request.buildResult().as(ShadedBuildResponse.class); + String shadedJarPath = buildResult.shadedJarPath(); + return shadedJarPath == null ? null : new File(shadedJarPath); + } + + /** Returns whether the target Flink version safely supports closing packaged programs. */ + public static boolean canSafelyClosePackagedProgram(SubmitRequest request) { + String[] parts = request.flinkVersion().version().split("\\."); + if (parts.length < 3) { + return false; + } + try { + int major = Integer.parseInt(parts[0].trim()); + int minor = Integer.parseInt(parts[1].trim()); + int patch = Integer.parseInt(parts[2].trim()); + return major >= 1 && (minor > 12 || (minor == 12 && patch >= 2)); + } catch (NumberFormatException ignored) { + return false; + } + } + + /** Verifies that the application artifact was built successfully before submission. */ + public static void validateBuildResult(SubmitRequest request) { + BuildResult buildResult = request.buildResult(); + String applicationName = effectiveApplicationName(request); + String target = + FlinkDeployMode.isKubernetesMode(request.deployMode()) + ? ", clusterId=" + + request.clusterId() + + ", namespace=" + + request.kubernetesNamespace() + : ""; + AssertUtils.required( + buildResult != null, + "[flink-submit] current job " + + applicationName + + " was not yet built; build result is empty" + + target); + AssertUtils.required( + buildResult.pass(), + "[flink-submit] current job " + applicationName + " build failed" + target); + } + + private static Map getParameters(SubmitRequest request, String prefix) { + if (request.appConf() == null) { + return Collections.emptyMap(); + } + return filterByPrefix(loadApplicationConfig(request.appConf()), prefix); + } + + private static Map loadApplicationConfig(String appConf) { + if (appConf.length() < CONFIG_SCHEME_LENGTH) { + throw invalidConfigFormat(appConf); + } + String scheme = appConf.substring(0, CONFIG_SCHEME_LENGTH); + switch (scheme) { + case "json://": + return parseJson(appConf.substring(CONFIG_SCHEME_LENGTH)); + case "yaml://": + return parseConfig(decompress(appConf), ConfigurationFormat.YAML, "inline YAML"); + case "conf://": + return parseConfig(decompress(appConf), ConfigurationFormat.HOCON, "inline HOCON"); + case "prop://": + return parseConfig( + decompress(appConf), ConfigurationFormat.PROPERTIES, "inline properties"); + case "hdfs://": + return parseHdfs(appConf); + default: + throw invalidConfigFormat(appConf); + } + } + + private static String decompress(String appConf) { + return DeflaterUtils.unzipString(appConf.trim().substring(CONFIG_SCHEME_LENGTH)); + } + + private static Map parseJson(String json) { + try { + Map values = + new ObjectMapper() + .readValue(json, new TypeReference<>() { + }); + return values.entrySet().stream() + .filter(entry -> entry.getValue() != null) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } catch (Exception e) { + throw new IllegalArgumentException("Failed to parse JSON application configuration", e); + } + } + + private static Map parseHdfs(String appConf) { + try { + String text = HdfsUtils.read(appConf); + String extension = appConf.substring(appConf.lastIndexOf('.') + 1).toLowerCase(); + switch (extension) { + case "yml": + case "yaml": + return parseConfig(text, ConfigurationFormat.YAML, appConf); + case "conf": + return parseConfig(text, ConfigurationFormat.HOCON, appConf); + case "properties": + return parseConfig(text, ConfigurationFormat.PROPERTIES, appConf); + default: + throw new IllegalArgumentException( + "HDFS application configuration must be YAML, HOCON, or properties"); + } + } catch (IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new IllegalStateException( + "Failed to read HDFS application configuration: " + appConf, e); + } + } + + private static Map parseConfig( + String content, + ConfigurationFormat format, + String origin) { + return ConfigurationParser.parse(content, format, origin).toMap(); + } + + private static Map filterByPrefix( + Map values, String prefix) { + Map result = new HashMap<>(); + for (Map.Entry entry : values.entrySet()) { + String value = entry.getValue(); + if (entry.getKey().startsWith(prefix) && value != null && !value.isEmpty()) { + result.put(entry.getKey().substring(prefix.length()), value); + } + } + return result; + } + + private static IllegalArgumentException invalidConfigFormat(String appConf) { + return new IllegalArgumentException( + "Unsupported application configuration format: " + appConf); + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/FlinkConfigurationOps.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationOps.java similarity index 68% rename from streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/FlinkConfigurationOps.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationOps.java index 58a99bb387..93cdec3b0c 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/FlinkConfigurationOps.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationOps.java @@ -15,18 +15,22 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.impl; +package org.apache.streampark.flink.client.configuration; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.Configuration; -/** Configuration helpers mirroring {@code FlinkClientTrait.EnhanceFlinkConfiguration#safeSet}. */ -final class FlinkConfigurationOps { +/** Null-safe operations for Flink configuration values. */ +public final class FlinkConfigurationOps { private FlinkConfigurationOps() { } - static Configuration safeSet(Configuration flinkConfig, ConfigOption option, T value) { + /** Sets a non-null, non-empty value and leaves the configuration unchanged otherwise. */ + public static Configuration setIfPresent( + Configuration flinkConfig, + ConfigOption option, + T value) { if (value != null && !value.toString().isEmpty()) { flinkConfig.set(option, value); } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/conf/FlinkSavepointOptions.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkSavepointOptions.java similarity index 93% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/conf/FlinkSavepointOptions.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkSavepointOptions.java index ee5ba6f4ed..c1709d312b 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/conf/FlinkSavepointOptions.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkSavepointOptions.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.conf; +package org.apache.streampark.flink.client.configuration; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; @@ -33,8 +33,8 @@ * that entirely: a {@code Configuration} is keyed by string, so a locally declared option addresses * exactly the same setting as Flink's own. * - *

The keys are part of Flink's public configuration surface, so they are as stable as the user's - * own {@code flink-conf.yaml} entries. + *

The keys are part of Flink's public configuration surface and map directly to the user's + * native Flink configuration entries. */ public final class FlinkSavepointOptions { diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java new file mode 100644 index 0000000000..9b2940e76e --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java @@ -0,0 +1,409 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.impl; + +import org.apache.streampark.common.configuration.Workspace; +import org.apache.streampark.common.enums.FlinkDeployMode; +import org.apache.streampark.common.util.AssertUtils; +import org.apache.streampark.common.util.ExceptionUtils; +import org.apache.streampark.common.util.LoggerSupport; +import org.apache.streampark.common.util.Utils; +import org.apache.streampark.flink.client.bean.FlinkJobGraphBuilder; +import org.apache.streampark.flink.client.bean.SubmissionConfigurationBuilder; +import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.SavepointRequest; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.SavepointResponse; +import org.apache.streampark.flink.client.response.SubmitResponse; +import org.apache.streampark.flink.core.FlinkClusterClient; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.flink.api.common.JobID; +import org.apache.flink.client.cli.CliArgsException; +import org.apache.flink.client.program.ClusterClient; +import org.apache.flink.client.program.PackagedProgram; +import org.apache.flink.configuration.CheckpointingOptions; +import org.apache.flink.configuration.ConfigOptions; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.runtime.jobgraph.JobGraph; +import org.apache.flink.util.FlinkException; + +import java.io.File; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** Coordinates the common lifecycle of Flink deployment clients. */ +public abstract class AbstractFlinkClient extends LoggerSupport { + + /** Submits a job with the deployment-mode-specific client. */ + public final SubmitResponse submit(SubmitRequest request) throws FlinkException { + logSubmitRequest(request); + Configuration configuration = + callAsFlinkException(() -> SubmissionConfigurationBuilder.build(request)); + setConfig(request, configuration); + return callAsFlinkException( + () -> doSubmit(request, configuration), + error -> logSubmitFailure(request, error)); + } + + /** Triggers a savepoint for an existing job. */ + public final SavepointResponse triggerSavepoint(TriggerSavepointRequest request) throws FlinkException { + logSavepointRequest("trigger savepoint", request); + return callAsFlinkException( + () -> doTriggerSavepoint(request, new Configuration())); + } + + /** Cancels an existing job, optionally creating a savepoint first. */ + public final CancelResponse cancel(CancelRequest request) throws FlinkException { + logSavepointRequest("cancel", request); + return callAsFlinkException(() -> doCancel(request, new Configuration())); + } + + /** Adds configuration required by a deployment mode. */ + protected abstract void setConfig(SubmitRequest request, Configuration configuration); + + /** Performs the deployment-mode-specific submission. */ + protected abstract SubmitResponse doSubmit( + SubmitRequest request, + Configuration configuration) throws FlinkException; + + /** Performs the deployment-mode-specific savepoint operation. */ + protected abstract SavepointResponse doTriggerSavepoint( + TriggerSavepointRequest request, + Configuration configuration) throws FlinkException; + + /** Performs the deployment-mode-specific cancellation. */ + protected abstract CancelResponse doCancel( + CancelRequest request, + Configuration configuration) throws FlinkException; + + /** Converts infrastructure exceptions to the stable Flink client exception contract. */ + protected static FlinkException asFlinkException(Throwable throwable) { + if (throwable instanceof FlinkException) { + return (FlinkException) throwable; + } + if (throwable instanceof Exception) { + return new FlinkException((Exception) throwable); + } + return new FlinkException(throwable.getMessage(), throwable); + } + + protected static T callAsFlinkException(FlinkCallable callable) throws FlinkException { + return callAsFlinkException(callable, ignored -> { + }); + } + + protected static T callAsFlinkException( + FlinkCallable callable, + Consumer onFailure) throws FlinkException { + try { + return callable.call(); + } catch (FlinkException e) { + onFailure.accept(e); + throw e; + } catch (Exception e) { + onFailure.accept(e); + throw asFlinkException(e); + } + } + + protected static T callAsFlinkExceptionMapping( + FlinkCallable callable, + Function exceptionMapper) throws FlinkException { + try { + return callable.call(); + } catch (FlinkException e) { + throw e; + } catch (Exception e) { + throw exceptionMapper.apply(e); + } + } + + protected final CancelResponse toCancelResponse( + CancelRequest request, + JobID jobId, + ClusterClient client) throws FlinkException { + return callAsFlinkException(() -> new CancelResponse(cancelJob(request, jobId, client))); + } + + protected final SavepointResponse toSavepointResponse( + TriggerSavepointRequest request, + JobID jobId, + ClusterClient client) throws FlinkException { + return callAsFlinkException( + () -> new SavepointResponse(triggerSavepoint(request, jobId, client))); + } + + protected final SubmitResponse submitJobGraphToCluster( + SubmitRequest request, + Configuration configuration, + File jarFile, + FlinkCallable> clientSupplier, + FlinkCallable clusterIdSupplier, + AutoCloseable... extraResources) throws FlinkException { + return callAsFlinkException( + () -> { + FlinkJobGraphBuilder.Result result = buildJobGraph(configuration, request, jarFile); + ClusterClient client = null; + try { + client = clientSupplier.call(); + String jobId = submitJobGraph(client, result.jobGraph()); + return new SubmitResponse( + clusterIdSupplier.call(), + configuration.toMap(), + jobId, + client.getWebInterfaceURL()); + } finally { + AutoCloseable[] resources = new AutoCloseable[extraResources.length + 2]; + resources[0] = result.program(); + resources[1] = client; + System.arraycopy(extraResources, 0, resources, 2, extraResources.length); + closeSubmissionResources(request, resources); + } + }); + } + + /** Tries JobGraph submission before falling back to the session REST API. */ + protected final SubmitResponse trySubmit( + SubmitRequest request, + Configuration configuration, + File jarFile, + SubmitFunction jobGraphSubmit, + SubmitFunction restApiSubmit) throws FlinkException { + try { + logInfo("[flink-submit] Submitting with the JobGraph protocol"); + return jobGraphSubmit.apply(request, configuration, jarFile); + } catch (FlinkException jobGraphFailure) { + logWarn( + "[flink-submit] JobGraph submission failed; trying the REST API: " + + ExceptionUtils.stringifyException(jobGraphFailure)); + try { + return restApiSubmit.apply(request, configuration, jarFile); + } catch (FlinkException restFailure) { + restFailure.addSuppressed(jobGraphFailure); + throw new FlinkException( + "Job submission failed with both the JobGraph protocol and the REST API", + restFailure); + } + } + } + + protected final FlinkJobGraphBuilder.Result buildJobGraph( + Configuration configuration, + SubmitRequest request, + File jarFile) throws Exception { + return FlinkJobGraphBuilder.build(configuration, request, jarFile); + } + + protected final JobID parseJobId(String jobId) throws CliArgsException { + try { + return JobID.fromHexString(jobId); + } catch (Exception e) { + throw new CliArgsException(e.getMessage()); + } + } + + protected final Configuration extractConfiguration( + String flinkHome, + Map properties) throws Exception { + return SubmissionConfigurationBuilder.extract(flinkHome, properties); + } + + protected final Configuration loadDefaultConfiguration(String flinkHome) { + return SubmissionConfigurationBuilder.loadDefault(flinkHome); + } + + protected final String cancelJob( + CancelRequest request, JobID jobId, ClusterClient client) throws Exception { + String savepointDirectory = resolveSavepointDirectory(request); + if (!request.withSavepoint() && !request.withDrain()) { + client.cancel(jobId).get(); + return null; + } + return new FlinkClusterClient<>(client) + .stopWithSavepoint( + jobId, request.withDrain(), savepointDirectory, request.nativeFormat()) + .get(); + } + + protected final String triggerSavepoint( + TriggerSavepointRequest request, + JobID jobId, + ClusterClient client) throws Exception { + return new FlinkClusterClient<>(client) + .triggerSavepoint( + jobId, resolveSavepointDirectory(request), request.nativeFormat()) + .get(); + } + + protected final void closeSubmissionResources( + SubmitRequest request, + AutoCloseable... resources) { + for (AutoCloseable resource : resources) { + if (resource == null) { + continue; + } + if (!(resource instanceof PackagedProgram) + || SubmitRequestResolver.canSafelyClosePackagedProgram(request)) { + Utils.close(resource); + } + } + } + + protected final void logEffectiveSubmitConfiguration(Configuration configuration) { + logInfo( + String.format( + "%n------------------------------------------------------------------%n" + + "Effective submit configuration: %s%n" + + "------------------------------------------------------------------%n", + configuration)); + } + + private String resolveSavepointDirectory(SavepointRequest request) { + if (!request.withSavepoint()) { + return null; + } + if (StringUtils.isNotBlank(request.savepointPath())) { + return request.savepointPath(); + } + + String defaultDirectory = + SubmissionConfigurationBuilder.getDefaultOption( + request.flinkVersion().getFlinkHome(), + ConfigOptions.key(CheckpointingOptions.SAVEPOINT_DIRECTORY.key()) + .stringType() + .defaultValue( + request.deployMode() == FlinkDeployMode.YARN_APPLICATION + ? Workspace.REMOTE.savepoints + : null)); + AssertUtils.required( + StringUtils.isNotBlank(defaultDirectory), + "Savepoint directory is not configured for " + request.deployMode().getName()); + return defaultDirectory; + } + + /** + * Invokes {@code ClusterClient.submitJob} across the Flink 1.x and 2.x signature change. + * + *

Flink 2.x accepts {@code ExecutionPlan}; Flink 1.x accepts {@code JobGraph}. Reflection is + * limited to this compatibility boundary so the rest of the client remains type-safe. + */ + private static String submitJobGraph(ClusterClient client, JobGraph jobGraph) throws Exception { + for (Method method : client.getClass().getMethods()) { + if ("submitJob".equals(method.getName()) + && method.getParameterCount() == 1 + && method.getParameterTypes()[0].isInstance(jobGraph)) { + Object future = method.invoke(client, jobGraph); + return ((CompletableFuture) future).get().toString(); + } + } + throw new FlinkException( + "No ClusterClient.submitJob method accepts JobGraph on " + + client.getClass().getName()); + } + + private void logSubmitRequest(SubmitRequest request) { + logInfo( + String.format( + "%n--------------------------------------- flink job start ---------------------------------------%n" + + " userFlinkHome : %s%n" + + " flinkVersion : %s%n" + + " appName : %s%n" + + " jobType : %s%n" + + " deployMode : %s%n" + + " k8sNamespace : %s%n" + + " flinkExposedType : %s%n" + + " clusterId : %s%n" + + " applicationType : %s%n" + + " savePoint : %s%n" + + " properties : %s%n" + + " args : %s%n" + + " appConf : %s%n" + + " flinkBuildResult : %s%n" + + "-------------------------------------------------------------------------------------------%n", + request.flinkVersion().getFlinkHome(), + request.flinkVersion().version(), + SubmitRequestResolver.effectiveApplicationName(request), + request.jobType(), + request.deployMode(), + request.kubernetesNamespace(), + request.flinkRestExposedType(), + request.clusterId(), + request.applicationType().getName(), + request.savePoint(), + formatProperties(request.properties()), + request.args(), + request.appConf(), + request.buildResult())); + } + + private void logSubmitFailure(SubmitRequest request, Exception error) { + logError( + "Flink job " + + SubmitRequestResolver.effectiveApplicationName(request) + + " failed to start in " + + request.deployMode().getName() + + ": " + + ExceptionUtils.stringifyException(error)); + } + + private void logSavepointRequest(String operation, SavepointRequest request) { + logInfo( + "Flink job " + + operation + + ": deployMode=" + + request.deployMode().getName() + + ", clusterId=" + + request.clusterId() + + ", jobId=" + + request.jobId() + + ", savepointPath=" + + request.savepointPath() + + ", nativeFormat=" + + request.nativeFormat()); + } + + private static String formatProperties(Map properties) { + if (MapUtils.isEmpty(properties)) { + return ""; + } + return properties.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining(" ")); + } + + @FunctionalInterface + protected interface FlinkCallable { + + T call() throws Exception; + } + + @FunctionalInterface + protected interface SubmitFunction { + + SubmitResponse apply(SubmitRequest request, Configuration configuration, File jarFile) throws FlinkException; + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractKubernetesNativeClient.java similarity index 74% rename from streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractKubernetesNativeClient.java index 98f267ec07..401deab92a 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/KubernetesNativeClientTrait.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractKubernetesNativeClient.java @@ -15,16 +15,18 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.trait; +package org.apache.streampark.flink.client.impl; import org.apache.streampark.common.enums.FlinkDeployMode; -import org.apache.streampark.common.enums.FlinkK8sRestExposedType; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.SavepointRequestTrait; -import org.apache.streampark.flink.client.bean.SavepointResponse; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; +import org.apache.streampark.common.enums.FlinkKubernetesRestExposedType; +import org.apache.streampark.common.util.Tuple2; +import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.SavepointRequest; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.SavepointResponse; import org.apache.streampark.flink.kubernetes.PodTemplateTool; import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse; @@ -45,18 +47,16 @@ import java.util.Map; -import scala.Tuple2; - -/** Kubernetes native mode submit support. */ -public abstract class KubernetesNativeClientTrait extends FlinkClientTrait { +/** Base implementation for Kubernetes native deployment clients. */ +public abstract class AbstractKubernetesNativeClient extends AbstractFlinkClient { @Override - public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - FlinkClientTrait.safeSet( + protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.CLUSTER_ID, submitRequest.clusterId()); - FlinkClientTrait.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.NAMESPACE, submitRequest.kubernetesNamespace()); - FlinkClientTrait.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.REST_SERVICE_EXPOSED_TYPE, covertToServiceExposedType(submitRequest.flinkRestExposedType())); @@ -69,21 +69,21 @@ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { if (podTemplatePaths != null) { for (Map.Entry entry : podTemplatePaths.entrySet()) { if (PodTemplateTool.KUBERNETES_POD_TEMPLATE.key().equals(entry.getKey())) { - FlinkClientTrait.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.KUBERNETES_POD_TEMPLATE, entry.getValue()); } else if (PodTemplateTool.KUBERNETES_JM_POD_TEMPLATE .key() .equals(entry.getKey())) { - FlinkClientTrait.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.JOB_MANAGER_POD_TEMPLATE, entry.getValue()); } else if (PodTemplateTool.KUBERNETES_TM_POD_TEMPLATE .key() .equals(entry.getKey())) { - FlinkClientTrait.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.TASK_MANAGER_POD_TEMPLATE, entry.getValue()); @@ -93,7 +93,7 @@ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { } if (!flinkConfig.contains(DeploymentOptionsInternal.CONF_DIR)) { - FlinkClientTrait.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, DeploymentOptionsInternal.CONF_DIR, submitRequest.flinkVersion().getFlinkHome() + "/conf"); @@ -107,11 +107,13 @@ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { } protected void setK8sDeployTarget(Configuration flinkConf, FlinkDeployMode deployMode) { - FlinkClientTrait.safeSet(flinkConf, DeploymentOptions.TARGET, deployMode.getName()); + FlinkConfigurationOps.setIfPresent(flinkConf, DeploymentOptions.TARGET, deployMode.getName()); } @Override - public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException { + protected CancelResponse doCancel( + CancelRequest cancelRequest, + Configuration flinkConfig) throws FlinkException { return executeClientAction( cancelRequest, flinkConfig, @@ -125,19 +127,19 @@ public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkC } @Override - public SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest savepointRequest, - Configuration flinkConfig) throws FlinkException { + protected SavepointResponse doTriggerSavepoint( + TriggerSavepointRequest savepointRequest, + Configuration flinkConfig) throws FlinkException { return executeClientAction( savepointRequest, flinkConfig, (jobId, clusterClient) -> toSavepointResponse(savepointRequest, jobId, clusterClient)); } - public O executeClientAction( - SavepointRequestTrait request, - Configuration flinkConfig, - ClientAction actFunc) throws FlinkException { + private O executeClientAction( + SavepointRequest request, + Configuration flinkConfig, + ClientAction actFunc) throws FlinkException { String hints = "[flink-client] execute " + request.getClass().getSimpleName() + " for flink job failed,"; if (StringUtils.isBlank(request.clusterId())) { @@ -147,9 +149,9 @@ public O executeClientAction( + flinkConfig.get(DeploymentOptions.TARGET)); } - FlinkClientTrait.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.CLUSTER_ID, request.clusterId()); - FlinkClientTrait.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.NAMESPACE, request.kubernetesNamespace()); KubernetesClusterDescriptor clusterDescriptor = null; @@ -186,7 +188,7 @@ public O executeClientAction( private void logClientActionFailure( String hints, Configuration flinkConfig, - SavepointRequestTrait request, + SavepointRequest request, Exception e) { logError( hints @@ -197,7 +199,7 @@ private void logClientActionFailure( e); } - public Tuple2 getK8sClusterDescriptorAndSpecification(Configuration flinkConfig) { + protected final Tuple2 getK8sClusterDescriptorAndSpecification(Configuration flinkConfig) { KubernetesClusterClientFactory clientFactory = new KubernetesClusterClientFactory(); KubernetesClusterDescriptor clusterDescriptor = clientFactory.createClusterDescriptor(flinkConfig); @@ -206,7 +208,8 @@ public Tuple2 getK8sClusterDe return new Tuple2<>(clusterDescriptor, clusterSpecification); } - public KubernetesClusterDescriptor getK8sClusterDescriptor(Configuration flinkConfig) { + protected final KubernetesClusterDescriptor getK8sClusterDescriptor( + Configuration flinkConfig) { KubernetesClusterClientFactory clientFactory = new KubernetesClusterClientFactory(); return clientFactory.createClusterDescriptor(flinkConfig); } @@ -221,7 +224,7 @@ protected String flinkConfIdentifierInfo(@Nonnull Configuration conf) { + conf.get(KubernetesConfigOptions.NAMESPACE); } - public String getDefaultKubernetesConf(String k8sConf) { + protected final String getDefaultKubernetesConf(String k8sConf) { String homePath = System.getProperty("user.home"); if (k8sConf != null) { return k8sConf.replace("~", homePath); @@ -229,21 +232,21 @@ public String getDefaultKubernetesConf(String k8sConf) { return homePath.concat("/.kube/config"); } - private ServiceExposedType covertToServiceExposedType(FlinkK8sRestExposedType exposedType) { - if (exposedType == FlinkK8sRestExposedType.CLUSTER_IP) { + private ServiceExposedType covertToServiceExposedType(FlinkKubernetesRestExposedType exposedType) { + if (exposedType == FlinkKubernetesRestExposedType.CLUSTER_IP) { return ServiceExposedType.ClusterIP; } - if (exposedType == FlinkK8sRestExposedType.NODE_PORT) { + if (exposedType == FlinkKubernetesRestExposedType.NODE_PORT) { return ServiceExposedType.NodePort; } - if (exposedType == FlinkK8sRestExposedType.LOAD_BALANCER) { + if (exposedType == FlinkKubernetesRestExposedType.LOAD_BALANCER) { return ServiceExposedType.LoadBalancer; } return ServiceExposedType.LoadBalancer; } @FunctionalInterface - protected interface ClientAction { + private interface ClientAction { O apply(JobID jobId, ClusterClient clusterClient) throws FlinkException; } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/YarnClientTrait.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractYarnClient.java similarity index 70% rename from streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/YarnClientTrait.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractYarnClient.java index 09e152a13d..3fc96283b9 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/YarnClientTrait.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractYarnClient.java @@ -15,16 +15,20 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.trait; +package org.apache.streampark.flink.client.impl; import org.apache.streampark.common.util.ExceptionUtils; import org.apache.streampark.common.util.HadoopUtils; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.SavepointRequestTrait; -import org.apache.streampark.flink.client.bean.SavepointResponse; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; +import org.apache.streampark.common.util.Tuple2; +import org.apache.streampark.common.util.Utils; +import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.SavepointRequest; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.SavepointResponse; import org.apache.flink.api.common.JobID; import org.apache.flink.client.deployment.ClusterSpecification; @@ -42,28 +46,28 @@ import java.lang.reflect.Method; import java.security.PrivilegedAction; -import scala.Tuple2; - -/** Yarn application mode submit support. */ -public abstract class YarnClientTrait extends FlinkClientTrait { +/** Base implementation for YARN deployment clients. */ +public abstract class AbstractYarnClient extends AbstractFlinkClient { private volatile Method deployInternalMethod; @Override - public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - FlinkClientTrait.safeSet( - flinkConfig, YarnConfigOptions.APPLICATION_NAME, submitRequest.effectiveAppName()); - FlinkClientTrait.safeSet( + protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { + FlinkConfigurationOps.setIfPresent( + flinkConfig, + YarnConfigOptions.APPLICATION_NAME, + SubmitRequestResolver.effectiveApplicationName(submitRequest)); + FlinkConfigurationOps.setIfPresent( flinkConfig, YarnConfigOptions.APPLICATION_TYPE, submitRequest.applicationType().getName()); - FlinkClientTrait.safeSet(flinkConfig, YarnConfigOptions.APPLICATION_TAGS, "streampark"); + FlinkConfigurationOps.setIfPresent(flinkConfig, YarnConfigOptions.APPLICATION_TAGS, "streampark"); } @Override - public SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest savepointRequest, - Configuration flinkConf) throws FlinkException { + protected SavepointResponse doTriggerSavepoint( + TriggerSavepointRequest savepointRequest, + Configuration flinkConf) throws FlinkException { return executeClientAction( savepointRequest, flinkConf, @@ -71,20 +75,23 @@ public SavepointResponse doTriggerSavepoint( } @Override - public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConf) throws FlinkException { + protected CancelResponse doCancel( + CancelRequest cancelRequest, + Configuration flinkConf) throws FlinkException { return executeClientAction( cancelRequest, flinkConf, (jobId, client) -> toCancelResponse(cancelRequest, jobId, client)); } - public ClusterClientProvider deployInternal( - YarnClusterDescriptor clusterDescriptor, - ClusterSpecification clusterSpecification, - String applicationName, - String yarnClusterEntrypoint, - JobGraph jobGraph, - Boolean detached) throws ReflectiveOperationException { + @SuppressWarnings("unchecked") + protected final ClusterClientProvider deployInternal( + YarnClusterDescriptor clusterDescriptor, + ClusterSpecification clusterSpecification, + String applicationName, + String yarnClusterEntrypoint, + JobGraph jobGraph, + Boolean detached) throws ReflectiveOperationException { Method method = getDeployInternalMethod(); return (ClusterClientProvider) method.invoke( clusterDescriptor, @@ -95,27 +102,27 @@ public ClusterClientProvider deployInternal( detached); } - public Tuple2 getYarnClusterDescriptor( - Configuration flinkConfig, - String user) throws FlinkException { + protected final Tuple2 getYarnClusterDescriptor( + Configuration flinkConfig, + String user) throws FlinkException { return accessYarnClusterDescriptor( user, () -> createYarnClusterDescriptor(flinkConfig)); } - public Tuple2 getYarnClusterDescriptor( - Configuration flinkConfig) throws FlinkException { + protected final Tuple2 getYarnClusterDescriptor( + Configuration flinkConfig) throws FlinkException { return getYarnClusterDescriptor(flinkConfig, ""); } - public Tuple2 getYarnClusterDeployDescriptor( - Configuration flinkConfig, - String user) throws FlinkException { + protected final Tuple2 getYarnClusterDeployDescriptor( + Configuration flinkConfig, + String user) throws FlinkException { return accessYarnClusterDescriptor( user, () -> createYarnClusterDeployDescriptor(flinkConfig)); } - public Tuple2 getYarnClusterDeployDescriptor( - Configuration flinkConfig) throws FlinkException { + protected final Tuple2 getYarnClusterDeployDescriptor( + Configuration flinkConfig) throws FlinkException { return getYarnClusterDeployDescriptor(flinkConfig, ""); } @@ -166,22 +173,27 @@ private Tuple2 createYarnClusterDep } private O executeClientAction( - SavepointRequestTrait request, Configuration flinkConf, + SavepointRequest request, Configuration flinkConf, ClientAction actionFunc) throws FlinkException { return callAsFlinkException( () -> { - JobID jobID = getJobID(request.jobId()); - FlinkClientTrait.safeSet(flinkConf, YarnConfigOptions.APPLICATION_ID, request.clusterId()); + JobID jobID = parseJobId(request.jobId()); + FlinkConfigurationOps.setIfPresent(flinkConf, YarnConfigOptions.APPLICATION_ID, request.clusterId()); Tuple2 descriptor = getYarnClusterDescriptor(flinkConf); - ClusterClient clusterClient = - descriptor._2().retrieve(descriptor._1()).getClusterClient(); - return applyClientAction(request, actionFunc, jobID, clusterClient); + ClusterClient clusterClient = null; + try { + clusterClient = + descriptor._2.retrieve(descriptor._1).getClusterClient(); + return applyClientAction(request, actionFunc, jobID, clusterClient); + } finally { + Utils.close(clusterClient, descriptor._2); + } }); } private O applyClientAction( - SavepointRequestTrait request, + SavepointRequest request, ClientAction actionFunc, JobID jobID, ClusterClient clusterClient) throws FlinkException { diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.java index b2e8f4e245..0c9b484ff5 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.java @@ -18,13 +18,15 @@ package org.apache.streampark.flink.client.impl; import org.apache.streampark.common.enums.FlinkDeployMode; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.SavepointResponse; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; -import org.apache.streampark.flink.client.trait.KubernetesNativeClientTrait; +import org.apache.streampark.common.util.Tuple2; +import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.SavepointResponse; +import org.apache.streampark.flink.client.response.SubmitResponse; import org.apache.streampark.flink.packer.pipeline.DockerImageBuildResponse; import org.apache.commons.lang3.StringUtils; @@ -40,10 +42,8 @@ import com.google.common.collect.Lists; -import scala.Tuple2; - /** Kubernetes native application mode submit. */ -public final class KubernetesNativeApplicationClient extends KubernetesNativeClientTrait { +public final class KubernetesNativeApplicationClient extends AbstractKubernetesNativeClient { public static final KubernetesNativeApplicationClient INSTANCE = new KubernetesNativeApplicationClient(); @@ -52,7 +52,9 @@ private KubernetesNativeApplicationClient() { } @Override - public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException { + protected SubmitResponse doSubmit(SubmitRequest submitRequest, + Configuration flinkConfig) throws FlinkException { + if (StringUtils.isBlank(submitRequest.clusterId())) { throw new IllegalArgumentException( String.format( @@ -62,56 +64,56 @@ public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkC return callAsFlinkException( () -> { - submitRequest.checkBuildResult(); + SubmitRequestResolver.validateBuildResult(submitRequest); DockerImageBuildResponse buildResult = (DockerImageBuildResponse) submitRequest.buildResult(); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, PipelineOptions.JARS, Lists.newArrayList(buildResult.dockerInnerMainJarPath())); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.CONTAINER_IMAGE, buildResult.flinkImageTag()); Tuple2 descriptorAndSpec = getK8sClusterDescriptorAndSpecification(flinkConfig); - KubernetesClusterDescriptor clusterDescriptor = descriptorAndSpec._1(); - ClusterSpecification clusterSpecification = descriptorAndSpec._2(); - - ApplicationConfiguration applicationConfig = - ApplicationConfiguration.fromConfiguration(flinkConfig); - ClusterClient clusterClient = - clusterDescriptor - .deployApplicationCluster(clusterSpecification, applicationConfig) - .getClusterClient(); - - String clusterId = clusterClient.getClusterId(); - SubmitResponse result = - new SubmitResponse( - clusterId, + KubernetesClusterDescriptor clusterDescriptor = descriptorAndSpec._1; + ClusterSpecification clusterSpecification = descriptorAndSpec._2; + ClusterClient clusterClient = null; + try { + ApplicationConfiguration applicationConfig = + ApplicationConfiguration.fromConfiguration(flinkConfig); + + clusterClient = + clusterDescriptor + .deployApplicationCluster(clusterSpecification, applicationConfig) + .getClusterClient(); + + logInfo("[flink-submit] Flink job submitted. " + flinkConfIdentifierInfo(flinkConfig)); + return new SubmitResponse( + clusterClient.getClusterId(), flinkConfig.toMap(), submitRequest.jobId(), clusterClient.getWebInterfaceURL()); - logInfo( - "[flink-submit] flink job has been submitted. " - + flinkConfIdentifierInfo(flinkConfig)); - - closeSubmit(submitRequest, clusterDescriptor, clusterClient); - return result; + } finally { + closeSubmissionResources(submitRequest, clusterClient, clusterDescriptor); + } }); } @Override - public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConf) throws FlinkException { + protected CancelResponse doCancel( + CancelRequest cancelRequest, + Configuration flinkConf) throws FlinkException { setK8sDeployTarget(flinkConf, FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION); return super.doCancel(cancelRequest, flinkConf); } @Override - public SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest request, - Configuration flinkConf) throws FlinkException { + protected SavepointResponse doTriggerSavepoint( + TriggerSavepointRequest request, + Configuration flinkConf) throws FlinkException { setK8sDeployTarget(flinkConf, FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION); return super.doTriggerSavepoint(request, flinkConf); } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.java index e91ef15dce..3bfb3c5cb7 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.java @@ -18,23 +18,25 @@ package org.apache.streampark.flink.client.impl; import org.apache.streampark.common.enums.FlinkDeployMode; +import org.apache.streampark.common.util.Tuple2; import org.apache.streampark.common.util.Utils; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.DeployRequest; -import org.apache.streampark.flink.client.bean.DeployRequestTrait; -import org.apache.streampark.flink.client.bean.DeployResponse; -import org.apache.streampark.flink.client.bean.SavepointResponse; -import org.apache.streampark.flink.client.bean.ShutDownRequest; -import org.apache.streampark.flink.client.bean.ShutDownResponse; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; -import org.apache.streampark.flink.client.tool.FlinkSessionSubmitHelper; -import org.apache.streampark.flink.client.trait.KubernetesNativeClientTrait; +import org.apache.streampark.flink.client.bean.SessionClusterRestClient; +import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.ClusterRequest; +import org.apache.streampark.flink.client.request.DeployRequest; +import org.apache.streampark.flink.client.request.ShutdownRequest; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.DeployResponse; +import org.apache.streampark.flink.client.response.SavepointResponse; +import org.apache.streampark.flink.client.response.ShutdownResponse; +import org.apache.streampark.flink.client.response.SubmitResponse; import org.apache.streampark.flink.core.FlinkKubernetesClient; import org.apache.streampark.flink.kubernetes.KubernetesRetriever; -import org.apache.streampark.flink.kubernetes.enums.FlinkK8sDeployMode; +import org.apache.streampark.flink.kubernetes.enums.FlinkKubernetesDeployMode; import org.apache.streampark.flink.kubernetes.model.ClusterKey; import org.apache.commons.lang3.StringUtils; @@ -53,43 +55,38 @@ import java.io.File; -import scala.Tuple2; - /** Kubernetes native session mode submit. */ -public final class KubernetesNativeSessionClient extends KubernetesNativeClientTrait { +public final class KubernetesNativeSessionClient extends AbstractKubernetesNativeClient { - public static final KubernetesNativeSessionClient INSTANCE = - new KubernetesNativeSessionClient(); + public static final KubernetesNativeSessionClient INSTANCE = new KubernetesNativeSessionClient(); private KubernetesNativeSessionClient() { } @Override - public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException { + protected SubmitResponse doSubmit( + SubmitRequest submitRequest, + Configuration flinkConfig) throws FlinkException { if (StringUtils.isBlank(submitRequest.clusterId())) { throw new IllegalArgumentException( String.format( "[flink-submit] submit flink job failed, clusterId is null, mode=%s", flinkConfig.get(DeploymentOptions.TARGET))); } - return trySubmit( - submitRequest, - flinkConfig, - submitRequest.userJarFile(), - this::jobGraphSubmit, - this::restApiSubmit); + + return this.restApiSubmit(submitRequest, flinkConfig, SubmitRequestResolver.userJarFile(submitRequest)); } /** Submit flink session job via rest api. */ - public SubmitResponse restApiSubmit( - SubmitRequest submitRequest, - Configuration flinkConfig, - File fatJar) throws FlinkException { + private SubmitResponse restApiSubmit( + SubmitRequest submitRequest, + Configuration flinkConfig, + File fatJar) throws FlinkException { return callAsFlinkException( () -> { ClusterKey clusterKey = ClusterKey.builder() - .executeMode(FlinkK8sDeployMode.SESSION) + .executeMode(FlinkKubernetesDeployMode.SESSION) .namespace(submitRequest.kubernetesNamespace()) .clusterId(submitRequest.clusterId()) .build(); @@ -100,39 +97,16 @@ public SubmitResponse restApiSubmit( "[flink-submit] retrieve flink session rest url failed, clusterKey=" + clusterKey)); String jobId = - FlinkSessionSubmitHelper.submitViaRestApi(jmRestUrl, fatJar, flinkConfig); + SessionClusterRestClient.submit(jmRestUrl, fatJar, flinkConfig); return new SubmitResponse( clusterKey.clusterId(), flinkConfig.toMap(), jobId, jmRestUrl); }); } - /** Submit flink session job with building JobGraph via ClusterClient api. */ - public SubmitResponse jobGraphSubmit( - SubmitRequest submitRequest, - Configuration flinkConfig, - File jarFile) throws FlinkException { - return callAsFlinkException( - () -> { - SubmitResponse result = - submitJobGraphToCluster( - submitRequest, - flinkConfig, - jarFile, - () -> getK8sClusterDescriptor(flinkConfig) - .retrieve(flinkConfig.getString(KubernetesConfigOptions.CLUSTER_ID)) - .getClusterClient(), - () -> flinkConfig.getString(KubernetesConfigOptions.CLUSTER_ID)); - logInfo( - "[flink-submit] flink job has been submitted. " - + flinkConfIdentifierInfo(flinkConfig) - + ", jobId: " - + result.jobId()); - return result; - }); - } - @Override - public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException { + protected CancelResponse doCancel( + CancelRequest cancelRequest, + Configuration flinkConfig) throws FlinkException { setK8sDeployTarget(flinkConfig, FlinkDeployMode.KUBERNETES_NATIVE_SESSION); return super.doCancel(cancelRequest, flinkConfig); } @@ -156,10 +130,10 @@ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { deployRequest.flinkVersion().version(), deployRequest.deployMode().name(), deployRequest.clusterId(), - deployRequest.k8sParam().kubernetesNamespace(), - deployRequest.k8sParam().flinkRestExposedType(), - deployRequest.k8sParam().serviceAccount(), - deployRequest.k8sParam().flinkImage(), + deployRequest.kubernetesDeploySpec().kubernetesNamespace(), + deployRequest.kubernetesDeploySpec().flinkRestExposedType(), + deployRequest.kubernetesDeploySpec().serviceAccount(), + deployRequest.kubernetesDeploySpec().flinkImage(), deployRequest.properties())); Configuration flinkConfig = getFlinkK8sConfig(deployRequest); @@ -172,7 +146,7 @@ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { try { Tuple2 kubernetesClusterDescriptor = getK8sClusterDescriptorAndSpecification(flinkConfig); - clusterDescriptor = kubernetesClusterDescriptor._1(); + clusterDescriptor = kubernetesClusterDescriptor._1; FlinkKubernetesClient kubeClientWrapper = new FlinkKubernetesClient(kubeClient); if (kubeClientWrapper.getService(deployRequest.clusterId()).isPresent()) { @@ -181,7 +155,7 @@ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { } else { client = clusterDescriptor - .deploySessionCluster(kubernetesClusterDescriptor._2()) + .deploySessionCluster(kubernetesClusterDescriptor._2) .getClusterClient(); } return new DeployResponse( @@ -193,25 +167,25 @@ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { } } - public ShutDownResponse shutdown(ShutDownRequest shutDownRequest) throws Exception { + public ShutdownResponse shutdown(ShutdownRequest shutdownRequest) throws Exception { FlinkKubeClient kubeClient = null; try { - Configuration flinkConfig = getFlinkK8sConfig(shutDownRequest); + Configuration flinkConfig = getFlinkK8sConfig(shutdownRequest); kubeClient = FlinkKubeClientFactory.getInstance().fromConfiguration(flinkConfig, "client"); FlinkKubernetesClient kubeClientWrapper = new FlinkKubernetesClient(kubeClient); boolean stopAndCleanupState = - shutDownRequest.clusterId() != null - && kubeClientWrapper.getService(shutDownRequest.clusterId()).isPresent(); + shutdownRequest.clusterId() != null + && kubeClientWrapper.getService(shutdownRequest.clusterId()).isPresent(); if (stopAndCleanupState) { - kubeClient.stopAndCleanupCluster(shutDownRequest.clusterId()); - return new ShutDownResponse(shutDownRequest.clusterId()); + kubeClient.stopAndCleanupCluster(shutdownRequest.clusterId()); + return new ShutdownResponse(shutdownRequest.clusterId()); } return null; } catch (Exception e) { logError( - "shutdown flink session fail in " + shutDownRequest.deployMode() + " mode", e); + "shutdown flink session fail in " + shutdownRequest.deployMode() + " mode", e); throw e; } finally { Utils.close(kubeClient); @@ -219,49 +193,57 @@ public ShutDownResponse shutdown(ShutDownRequest shutDownRequest) throws Excepti } @Override - public SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest triggerSavepointRequest, - Configuration flinkConfig) throws FlinkException { - FlinkConfigurationOps.safeSet( + protected SavepointResponse doTriggerSavepoint( + TriggerSavepointRequest triggerSavepointRequest, + Configuration flinkConfig) throws FlinkException { + FlinkConfigurationOps.setIfPresent( flinkConfig, DeploymentOptions.TARGET, FlinkDeployMode.KUBERNETES_NATIVE_SESSION.getName()); return super.doTriggerSavepoint(triggerSavepointRequest, flinkConfig); } - private Configuration getFlinkK8sConfig(DeployRequestTrait deployRequest) throws Exception { - Configuration flinkConfig = - extractConfiguration( - deployRequest.flinkVersion().getFlinkHome(), deployRequest.properties()); - FlinkConfigurationOps.safeSet( - flinkConfig, DeploymentOptions.TARGET, KubernetesDeploymentTarget.SESSION.getName()); - FlinkConfigurationOps.safeSet( + private Configuration getFlinkK8sConfig(ClusterRequest deployRequest) throws Exception { + Configuration flinkConfig = extractConfiguration( + deployRequest.flinkVersion().getFlinkHome(), deployRequest.properties()); + + FlinkConfigurationOps.setIfPresent(flinkConfig, DeploymentOptions.TARGET, + KubernetesDeploymentTarget.SESSION.getName()); + + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.NAMESPACE, - deployRequest.k8sParam().kubernetesNamespace()); - FlinkConfigurationOps.safeSet( + deployRequest.kubernetesDeploySpec().kubernetesNamespace()); + + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT, - deployRequest.k8sParam().serviceAccount()); - FlinkConfigurationOps.safeSet( + deployRequest.kubernetesDeploySpec().serviceAccount()); + + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.CLUSTER_ID, deployRequest.clusterId()); - FlinkConfigurationOps.safeSet( + + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.CONTAINER_IMAGE, - deployRequest.k8sParam().flinkImage()); - FlinkConfigurationOps.safeSet( + deployRequest.kubernetesDeploySpec().flinkImage()); + + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.REST_SERVICE_EXPOSED_TYPE, ServiceExposedType.valueOf( - deployRequest.k8sParam().flinkRestExposedType().getName())); - FlinkConfigurationOps.safeSet( + deployRequest.kubernetesDeploySpec().flinkRestExposedType().getName())); + + FlinkConfigurationOps.setIfPresent( flinkConfig, KubernetesConfigOptions.KUBE_CONFIG_FILE, - getDefaultKubernetesConf(deployRequest.k8sParam().kubeConf())); - FlinkConfigurationOps.safeSet( + getDefaultKubernetesConf(deployRequest.kubernetesDeploySpec().kubeConf())); + + FlinkConfigurationOps.setIfPresent( flinkConfig, DeploymentOptionsInternal.CONF_DIR, deployRequest.flinkVersion().getFlinkHome() + "/conf"); + return flinkConfig; } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/LocalClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/LocalClient.java index 9df26a32eb..5be4d82b65 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/LocalClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/LocalClient.java @@ -17,17 +17,18 @@ package org.apache.streampark.flink.client.impl; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.SavepointResponse; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; -import org.apache.streampark.flink.client.trait.FlinkClientTrait; +import org.apache.streampark.flink.client.bean.FlinkJobGraphBuilder; +import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.SavepointResponse; +import org.apache.streampark.flink.client.response.SubmitResponse; import org.apache.flink.client.deployment.executors.RemoteExecutor; import org.apache.flink.client.program.MiniClusterClient; -import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.DeploymentOptions; @@ -35,15 +36,12 @@ import org.apache.flink.configuration.PipelineOptions; import org.apache.flink.configuration.RestOptions; import org.apache.flink.configuration.TaskManagerOptions; -import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.runtime.minicluster.MiniCluster; import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; import org.apache.flink.util.FlinkException; -import scala.Tuple2; - /** Submit Flink jobs to a local mini cluster. */ -public final class LocalClient extends FlinkClientTrait { +public final class LocalClient extends AbstractFlinkClient { public static final LocalClient INSTANCE = new LocalClient(); @@ -51,42 +49,52 @@ private LocalClient() { } @Override - public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - FlinkConfigurationOps.safeSet(flinkConfig, PipelineOptions.NAME, submitRequest.effectiveAppName()); + protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { + FlinkConfigurationOps.setIfPresent( + flinkConfig, + PipelineOptions.NAME, + SubmitRequestResolver.effectiveApplicationName(submitRequest)); logEffectiveSubmitConfiguration(flinkConfig); } @Override - public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException { + protected SubmitResponse doSubmit( + SubmitRequest submitRequest, + Configuration flinkConfig) throws FlinkException { return callAsFlinkException( () -> { - Tuple2 programJobGraph = - getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile()); - PackagedProgram packageProgram = programJobGraph._1(); - JobGraph jobGraph = programJobGraph._2(); - MiniClusterClient client = createLocalCluster(flinkConfig); - String jobId = client.submitJob(jobGraph).get().toString(); - SubmitResponse resp = - new SubmitResponse(jobId, flinkConfig.toMap(), jobId, client.getWebInterfaceURL()); - closeSubmit(submitRequest, packageProgram, client); - return resp; + FlinkJobGraphBuilder.Result job = buildJobGraph( + flinkConfig, + submitRequest, + SubmitRequestResolver.userJarFile(submitRequest)); + MiniClusterClient client = null; + try { + client = createLocalCluster(flinkConfig); + String jobId = client.submitJob(job.jobGraph()).get().toString(); + return new SubmitResponse( + jobId, flinkConfig.toMap(), jobId, client.getWebInterfaceURL()); + } finally { + closeSubmissionResources(submitRequest, job.program(), client); + } }); } @Override - public SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest savepointRequest, - Configuration flinkConfig) throws FlinkException { + protected SavepointResponse doTriggerSavepoint( + TriggerSavepointRequest savepointRequest, + Configuration flinkConfig) throws FlinkException { return RemoteClient.INSTANCE.doTriggerSavepoint(savepointRequest, flinkConfig); } @Override - public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException { + protected CancelResponse doCancel( + CancelRequest cancelRequest, + Configuration flinkConfig) throws FlinkException { return RemoteClient.INSTANCE.doCancel(cancelRequest, flinkConfig); } private MiniClusterClient createLocalCluster(Configuration flinkConfig) throws Exception { - FlinkConfigurationOps.safeSet(flinkConfig, JobManagerOptions.PORT, 0); + FlinkConfigurationOps.setIfPresent(flinkConfig, JobManagerOptions.PORT, 0); int numTaskManagers = flinkConfig.getInteger( @@ -107,11 +115,11 @@ private MiniClusterClient createLocalCluster(Configuration flinkConfig) throws E String host = "localhost"; int port = cluster.getRestAddress().get().getPort(); - FlinkConfigurationOps.safeSet(flinkConfig, JobManagerOptions.ADDRESS, host); - FlinkConfigurationOps.safeSet(flinkConfig, JobManagerOptions.PORT, port); - FlinkConfigurationOps.safeSet(flinkConfig, RestOptions.ADDRESS, host); - FlinkConfigurationOps.safeSet(flinkConfig, RestOptions.PORT, port); - FlinkConfigurationOps.safeSet(flinkConfig, DeploymentOptions.TARGET, RemoteExecutor.NAME); + FlinkConfigurationOps.setIfPresent(flinkConfig, JobManagerOptions.ADDRESS, host); + FlinkConfigurationOps.setIfPresent(flinkConfig, JobManagerOptions.PORT, port); + FlinkConfigurationOps.setIfPresent(flinkConfig, RestOptions.ADDRESS, host); + FlinkConfigurationOps.setIfPresent(flinkConfig, RestOptions.PORT, port); + FlinkConfigurationOps.setIfPresent(flinkConfig, DeploymentOptions.TARGET, RemoteExecutor.NAME); logInfo(String.format("%nStarting local Flink cluster (host: localhost, port: %d).%n", port)); diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/RemoteClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/RemoteClient.java index 7050dceda0..5c3e6dbd48 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/RemoteClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/RemoteClient.java @@ -18,15 +18,17 @@ package org.apache.streampark.flink.client.impl; import org.apache.streampark.common.util.ClassLoaderUtils; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.SavepointRequestTrait; -import org.apache.streampark.flink.client.bean.SavepointResponse; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; -import org.apache.streampark.flink.client.tool.FlinkSessionSubmitHelper; -import org.apache.streampark.flink.client.trait.FlinkClientTrait; +import org.apache.streampark.common.util.Tuple2; +import org.apache.streampark.flink.client.bean.SessionClusterRestClient; +import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.SavepointRequest; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.SavepointResponse; +import org.apache.streampark.flink.client.response.SubmitResponse; import org.apache.flink.api.common.JobID; import org.apache.flink.client.deployment.ClusterClientFactory; @@ -41,10 +43,8 @@ import java.io.File; -import scala.Tuple2; - /** Submit Flink jobs to a remote standalone cluster. */ -public final class RemoteClient extends FlinkClientTrait { +public final class RemoteClient extends AbstractFlinkClient { public static final RemoteClient INSTANCE = new RemoteClient(); @@ -52,23 +52,27 @@ private RemoteClient() { } @Override - public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { + protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { // no extra configuration } @Override - public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException { + protected SubmitResponse doSubmit( + SubmitRequest submitRequest, + Configuration flinkConfig) throws FlinkException { return callAsFlinkException( () -> trySubmit( submitRequest, flinkConfig, - submitRequest.userJarFile(), + SubmitRequestResolver.userJarFile(submitRequest), this::jobGraphSubmit, this::restApiSubmit)); } @Override - public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException { + protected CancelResponse doCancel( + CancelRequest cancelRequest, + Configuration flinkConfig) throws FlinkException { return executeClientAction( cancelRequest, flinkConfig, @@ -76,9 +80,9 @@ public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkC } @Override - public SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest savepointRequest, - Configuration flinkConfig) throws FlinkException { + protected SavepointResponse doTriggerSavepoint( + TriggerSavepointRequest savepointRequest, + Configuration flinkConfig) throws FlinkException { return executeClientAction( savepointRequest, flinkConfig, @@ -86,62 +90,63 @@ public SavepointResponse doTriggerSavepoint( } /** Submit flink session job via rest api. */ - public SubmitResponse restApiSubmit( - SubmitRequest submitRequest, - Configuration flinkConfig, - File fatJar) throws FlinkException { + private SubmitResponse restApiSubmit( + SubmitRequest submitRequest, + Configuration flinkConfig, + File fatJar) throws FlinkException { return callAsFlinkException( () -> { Tuple2 standAloneDescriptor = getStandAloneClusterDescriptor(flinkConfig); - StandaloneClusterId yarnClusterId = standAloneDescriptor._1(); - StandaloneClusterDescriptor clusterDescriptor = standAloneDescriptor._2(); - - ClusterClient client = - clusterDescriptor.retrieve(yarnClusterId).getClusterClient(); - String jobId = - FlinkSessionSubmitHelper.submitViaRestApi( - client.getWebInterfaceURL(), fatJar, flinkConfig); - logInfo( - String.format( - "%s mode submit by restApi, WebInterfaceURL %s, jobId: %s", - submitRequest.deployMode(), client.getWebInterfaceURL(), jobId)); - SubmitResponse resp = - new SubmitResponse(null, flinkConfig.toMap(), jobId, client.getWebInterfaceURL()); - closeSubmit(submitRequest, client, clusterDescriptor); - return resp; + StandaloneClusterId yarnClusterId = standAloneDescriptor._1; + StandaloneClusterDescriptor clusterDescriptor = standAloneDescriptor._2; + ClusterClient client = null; + try { + client = clusterDescriptor.retrieve(yarnClusterId).getClusterClient(); + String jobId = + SessionClusterRestClient.submit( + client.getWebInterfaceURL(), fatJar, flinkConfig); + logInfo( + String.format( + "%s mode submit by REST API, WebInterfaceURL %s, jobId: %s", + submitRequest.deployMode(), client.getWebInterfaceURL(), jobId)); + return new SubmitResponse( + null, flinkConfig.toMap(), jobId, client.getWebInterfaceURL()); + } finally { + closeSubmissionResources(submitRequest, client, clusterDescriptor); + } }); } /** Submit flink session job with building JobGraph via Standalone ClusterClient api. */ - public SubmitResponse jobGraphSubmit( - SubmitRequest submitRequest, - Configuration flinkConfig, - File jarFile) throws FlinkException { + private SubmitResponse jobGraphSubmit( + SubmitRequest submitRequest, + Configuration flinkConfig, + File jarFile) throws FlinkException { Tuple2 standAloneDescriptor = getStandAloneClusterDescriptor(flinkConfig); return submitJobGraphToCluster( submitRequest, flinkConfig, jarFile, - () -> standAloneDescriptor._2().retrieve(standAloneDescriptor._1()).getClusterClient(), + () -> standAloneDescriptor._2.retrieve(standAloneDescriptor._1).getClusterClient(), () -> null, - standAloneDescriptor._2()); + standAloneDescriptor._2); } - private O executeClientAction( - R request, - Configuration flinkConfig, - ClientAction actFunc) throws FlinkException { + private O executeClientAction( + R request, + Configuration flinkConfig, + ClientAction actFunc) throws FlinkException { return callAsFlinkException( () -> { - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, DeploymentOptions.TARGET, request.deployMode().getName()); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, RestOptions.ADDRESS, request.properties().get(RestOptions.ADDRESS.key()).toString()); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, RestOptions.PORT, Integer.parseInt( @@ -150,12 +155,12 @@ private O executeClientAction( Tuple2 descriptor = getStandAloneClusterDescriptor(flinkConfig); ClusterClient clusterClient = - descriptor._2().retrieve(descriptor._1()).getClusterClient(); + descriptor._2.retrieve(descriptor._1).getClusterClient(); try { return actFunc.apply(JobID.fromHexString(request.jobId()), clusterClient); } finally { clusterClient.close(); - descriptor._2().close(); + descriptor._2.close(); } }, e -> { diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java index 552a4e683f..c24f5f2e06 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java @@ -17,17 +17,19 @@ package org.apache.streampark.flink.client.impl; -import org.apache.streampark.common.conf.Workspace; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.enums.FlinkJobType; import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.FileUtils; import org.apache.streampark.common.util.HdfsUtils; -import org.apache.streampark.flink.client.bean.HdfsWorkspace; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.client.trait.YarnClientTrait; +import org.apache.streampark.common.util.Tuple2; +import org.apache.streampark.flink.client.bean.RemoteWorkspace; +import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.response.SubmitResponse; import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse; import org.apache.flink.client.deployment.ClusterSpecification; @@ -41,37 +43,36 @@ import org.apache.flink.yarn.configuration.YarnConfigOptions; import org.apache.hadoop.yarn.api.records.ApplicationId; +import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import scala.Tuple2; - /** Yarn application mode submit. */ -public final class YarnApplicationClient extends YarnClientTrait { +public final class YarnApplicationClient extends AbstractYarnClient { public static final YarnApplicationClient INSTANCE = new YarnApplicationClient(); - private static final Workspace WORKSPACE = Workspace.remote; + private static final Workspace WORKSPACE = Workspace.REMOTE; private YarnApplicationClient() { } @Override - public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { + protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { super.setConfig(submitRequest, flinkConfig); List providedLibs = new ArrayList<>(); - HdfsWorkspace hdfsWorkspace = submitRequest.hdfsWorkspace(); + RemoteWorkspace hdfsWorkspace = RemoteWorkspace.resolve(submitRequest.flinkVersion()); providedLibs.add(hdfsWorkspace.flinkLib()); providedLibs.add(hdfsWorkspace.flinkPlugins()); providedLibs.add(hdfsWorkspace.appJars()); if (submitRequest.jobType() == FlinkJobType.FLINK_SQL) { providedLibs.add( - WORKSPACE.APP_SHIMS() + "/flink-" + submitRequest.flinkVersion().majorVersion()); - String jobLib = WORKSPACE.APP_WORKSPACE() + "/" + submitRequest.id() + "/lib"; + WORKSPACE.shims + "/flink-" + submitRequest.flinkVersion().majorVersion()); + String jobLib = WORKSPACE.workspace + "/" + submitRequest.id() + "/lib"; try { if (HdfsUtils.exists(jobLib)) { providedLibs.add(jobLib); @@ -81,57 +82,57 @@ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { } } - FlinkConfigurationOps.safeSet(flinkConfig, YarnConfigOptions.PROVIDED_LIB_DIRS, providedLibs); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent(flinkConfig, YarnConfigOptions.PROVIDED_LIB_DIRS, providedLibs); + FlinkConfigurationOps.setIfPresent( flinkConfig, YarnConfigOptions.FLINK_DIST_JAR, hdfsWorkspace.flinkDistJar()); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, PipelineOptions.JARS, Collections.singletonList( ((ShadedBuildResponse) submitRequest.buildResult()).shadedJarPath())); - FlinkConfigurationOps.safeSet( - flinkConfig, YarnConfigOptions.APPLICATION_NAME, submitRequest.effectiveAppName()); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( + flinkConfig, + YarnConfigOptions.APPLICATION_NAME, + SubmitRequestResolver.effectiveApplicationName(submitRequest)); + FlinkConfigurationOps.setIfPresent( flinkConfig, YarnConfigOptions.APPLICATION_TYPE, submitRequest.applicationType().getName()); if (submitRequest.jobType() == FlinkJobType.PYFLINK) { - String pyVenv = WORKSPACE.APP_PYTHON_VENV(); + File userJar = SubmitRequestResolver.userJarFile(submitRequest); + AssertUtils.required(userJar != null, "PyFlink application archive is missing"); + String pyVenv = WORKSPACE.pythonVenv; AssertUtils.required(FsOperator.hdfs().exists(pyVenv), pyVenv + " File does not exist"); String localLib = - Workspace.local().APP_WORKSPACE() + "/" + submitRequest.id() + "/lib"; + Workspace.LOCAL.workspace + "/" + submitRequest.id() + "/lib"; if (FileUtils.exists(localLib) && FileUtils.directoryNotBlank(localLib)) { - FlinkConfigurationOps.safeSet(flinkConfig, PipelineOptions.JARS, Arrays.asList(localLib)); + FlinkConfigurationOps.setIfPresent(flinkConfig, PipelineOptions.JARS, Arrays.asList(localLib)); } ArrayList shipFiles = new ArrayList<>(); - shipFiles.add(submitRequest.userJarFile().getParentFile().getAbsolutePath()); + shipFiles.add(userJar.getParentFile().getAbsolutePath()); - FlinkConfigurationOps.safeSet(flinkConfig, YarnConfigOptions.SHIP_FILES, shipFiles); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent(flinkConfig, YarnConfigOptions.SHIP_FILES, shipFiles); + FlinkConfigurationOps.setIfPresent( flinkConfig, PythonOptions.PYTHON_FILES, - submitRequest.userJarFile().getParentFile().getName()); - FlinkConfigurationOps.safeSet(flinkConfig, PythonOptions.PYTHON_ARCHIVES, pyVenv); - FlinkConfigurationOps.safeSet( + userJar.getParentFile().getName()); + FlinkConfigurationOps.setIfPresent(flinkConfig, PythonOptions.PYTHON_ARCHIVES, pyVenv); + FlinkConfigurationOps.setIfPresent( flinkConfig, PythonOptions.PYTHON_CLIENT_EXECUTABLE, Constants.PYTHON_EXECUTABLE); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, PythonOptions.PYTHON_EXECUTABLE, Constants.PYTHON_EXECUTABLE); List args = flinkConfig.get(ApplicationConfiguration.APPLICATION_ARGS); ArrayList argsList = new ArrayList<>(args); argsList.add("-pym"); argsList.add( - submitRequest - .userJarFile() + userJar .getName() - .substring( - 0, - submitRequest.userJarFile().getName().length() - - Constants.PYTHON_SUFFIX.length())); - FlinkConfigurationOps.safeSet( + .substring(0, userJar.getName().length() - Constants.PYTHON_SUFFIX.length())); + FlinkConfigurationOps.setIfPresent( flinkConfig, ApplicationConfiguration.APPLICATION_ARGS, argsList); } @@ -139,29 +140,37 @@ public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { } @Override - public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException { + protected SubmitResponse doSubmit( + SubmitRequest submitRequest, + Configuration flinkConfig) throws FlinkException { return callAsFlinkException( () -> { Tuple2 deployDescriptor = getYarnClusterDeployDescriptor(flinkConfig, submitRequest.hadoopUser()); - ClusterSpecification clusterSpecification = deployDescriptor._1(); - YarnClusterDescriptor clusterDescriptor = deployDescriptor._2(); + ClusterSpecification clusterSpecification = deployDescriptor._1; + YarnClusterDescriptor clusterDescriptor = deployDescriptor._2; logClusterSpecification(clusterSpecification); - ApplicationConfiguration applicationConfiguration = - ApplicationConfiguration.fromConfiguration(flinkConfig); - ClusterClient clusterClient = - clusterDescriptor - .deployApplicationCluster(clusterSpecification, applicationConfiguration) - .getClusterClient(); - ApplicationId applicationId = clusterClient.getClusterId(); - String jobManagerUrl = clusterClient.getWebInterfaceURL(); - logYarnJobStarted(applicationId); - - SubmitResponse resp = - new SubmitResponse(applicationId.toString(), flinkConfig.toMap(), "", jobManagerUrl); - closeSubmit(submitRequest, clusterClient, clusterDescriptor); - return resp; + ClusterClient clusterClient = null; + try { + ApplicationConfiguration applicationConfiguration = + ApplicationConfiguration.fromConfiguration(flinkConfig); + clusterClient = + clusterDescriptor + .deployApplicationCluster( + clusterSpecification, applicationConfiguration) + .getClusterClient(); + ApplicationId applicationId = clusterClient.getClusterId(); + logYarnJobStarted(applicationId); + return new SubmitResponse( + applicationId.toString(), + flinkConfig.toMap(), + "", + clusterClient.getWebInterfaceURL()); + } finally { + closeSubmissionResources( + submitRequest, clusterClient, clusterDescriptor); + } }); } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnPerJobClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnPerJobClient.java index ac57e11ead..43b077f674 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnPerJobClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnPerJobClient.java @@ -17,16 +17,19 @@ package org.apache.streampark.flink.client.impl; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.client.trait.YarnClientTrait; +import org.apache.streampark.common.util.Tuple2; +import org.apache.streampark.common.util.Utils; +import org.apache.streampark.flink.client.bean.FlinkJobGraphBuilder; +import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.SubmitResponse; import org.apache.streampark.flink.util.FlinkUtils; import org.apache.flink.client.deployment.ClusterSpecification; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.DeploymentOptions; import org.apache.flink.runtime.jobgraph.JobGraph; @@ -37,10 +40,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ApplicationId; -import scala.Tuple2; - /** Yarn per-job mode submit. */ -public final class YarnPerJobClient extends YarnClientTrait { +public final class YarnPerJobClient extends AbstractYarnClient { public static final YarnPerJobClient INSTANCE = new YarnPerJobClient(); @@ -48,79 +49,86 @@ private YarnPerJobClient() { } @Override - public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { + protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { super.setConfig(submitRequest, flinkConfig); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.PER_JOB.getName()); - FlinkConfigurationOps.safeSet(flinkConfig, DeploymentOptions.ATTACHED, true); - FlinkConfigurationOps.safeSet(flinkConfig, DeploymentOptions.SHUTDOWN_IF_ATTACHED, true); + FlinkConfigurationOps.setIfPresent(flinkConfig, DeploymentOptions.ATTACHED, true); + FlinkConfigurationOps.setIfPresent(flinkConfig, DeploymentOptions.SHUTDOWN_IF_ATTACHED, true); logEffectiveSubmitConfiguration(flinkConfig); } @Override - public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException { + protected SubmitResponse doSubmit( + SubmitRequest submitRequest, + Configuration flinkConfig) throws FlinkException { + return callAsFlinkException( () -> { String flinkHome = submitRequest.flinkVersion().getFlinkHome(); Tuple2 deployDescriptor = getYarnClusterDeployDescriptor(flinkConfig, submitRequest.hadoopUser()); - ClusterSpecification clusterSpecification = deployDescriptor._1(); - YarnClusterDescriptor clusterDescriptor = deployDescriptor._2(); + ClusterSpecification clusterSpecification = deployDescriptor._1; + YarnClusterDescriptor clusterDescriptor = deployDescriptor._2; String flinkDistJar = FlinkUtils.getFlinkDistJar(flinkHome); clusterDescriptor.setLocalJarPath(new Path(flinkDistJar)); clusterDescriptor.addShipFiles(java.util.Collections.singletonList(new Path(flinkHome + "/lib"))); - PackagedProgram packagedProgram = null; - ClusterClient clusterClient; - logClusterSpecification(clusterSpecification); - - Tuple2 programJobGraph = - getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile()); - packagedProgram = programJobGraph._1(); - JobGraph jobGraph = programJobGraph._2(); - - logInfo( - String.format( - "%n-------------------------<>------------------------%n" - + "jobGraph getJobID: %s%n" - + "__________________________________________________________________%n", - jobGraph.getJobID())); - - clusterClient = - deployInternal( - clusterDescriptor, - clusterSpecification, - submitRequest.effectiveAppName(), - YarnJobClusterEntrypoint.class.getName(), - jobGraph, - true) - .getClusterClient(); - - ApplicationId applicationId = clusterClient.getClusterId(); - String jobManagerUrl = clusterClient.getWebInterfaceURL(); - logYarnJobStarted(applicationId); - - SubmitResponse resp = - new SubmitResponse(applicationId.toString(), flinkConfig.toMap(), "", jobManagerUrl); - closeSubmit(submitRequest, packagedProgram, clusterClient, clusterDescriptor); - return resp; + ClusterClient clusterClient = null; + FlinkJobGraphBuilder.Result job = null; + try { + logClusterSpecification(clusterSpecification); + job = + buildJobGraph( + flinkConfig, + submitRequest, + SubmitRequestResolver.userJarFile(submitRequest)); + JobGraph jobGraph = job.jobGraph(); + clusterClient = + deployInternal( + clusterDescriptor, + clusterSpecification, + SubmitRequestResolver.effectiveApplicationName(submitRequest), + YarnJobClusterEntrypoint.class.getName(), + jobGraph, + true) + .getClusterClient(); + ApplicationId applicationId = clusterClient.getClusterId(); + logYarnJobStarted(applicationId); + return new SubmitResponse( + applicationId.toString(), + flinkConfig.toMap(), + "", + clusterClient.getWebInterfaceURL()); + } finally { + closeSubmissionResources( + submitRequest, + job == null ? null : job.program(), + clusterClient, + clusterDescriptor); + } }); } @Override - public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException { + protected CancelResponse doCancel( + CancelRequest cancelRequest, + Configuration flinkConfig) throws FlinkException { return callAsFlinkException( () -> { CancelResponse response = super.doCancel(cancelRequest, flinkConfig); Tuple2 yarnClusterDescriptor = getYarnClusterDescriptor(flinkConfig); - yarnClusterDescriptor - ._2() - .killCluster(ApplicationId.fromString(cancelRequest.clusterId())); - return response; + try { + yarnClusterDescriptor._2 + .killCluster(ApplicationId.fromString(cancelRequest.clusterId())); + return response; + } finally { + Utils.close(yarnClusterDescriptor._2); + } }); } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java index 23744254be..6f65848847 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java @@ -17,28 +17,29 @@ package org.apache.streampark.flink.client.impl; +import org.apache.streampark.common.util.Tuple2; import org.apache.streampark.common.util.Utils; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.DeployRequest; -import org.apache.streampark.flink.client.bean.DeployResponse; -import org.apache.streampark.flink.client.bean.HdfsWorkspace; -import org.apache.streampark.flink.client.bean.SavepointResponse; -import org.apache.streampark.flink.client.bean.ShutDownRequest; -import org.apache.streampark.flink.client.bean.ShutDownResponse; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; -import org.apache.streampark.flink.client.trait.YarnClientTrait; +import org.apache.streampark.flink.client.bean.FlinkJobGraphBuilder; +import org.apache.streampark.flink.client.bean.RemoteWorkspace; +import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.DeployRequest; +import org.apache.streampark.flink.client.request.ShutdownRequest; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.response.CancelResponse; +import org.apache.streampark.flink.client.response.DeployResponse; +import org.apache.streampark.flink.client.response.SavepointResponse; +import org.apache.streampark.flink.client.response.ShutdownResponse; +import org.apache.streampark.flink.client.response.SubmitResponse; import org.apache.commons.lang3.StringUtils; import org.apache.flink.client.deployment.ClusterSpecification; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.DeploymentOptions; import org.apache.flink.configuration.DeploymentOptionsInternal; -import org.apache.flink.runtime.jobgraph.JobGraph; import org.apache.flink.util.FlinkException; import org.apache.flink.yarn.YarnClusterDescriptor; import org.apache.flink.yarn.configuration.YarnConfigOptions; @@ -51,10 +52,8 @@ import java.util.ArrayList; import java.util.Map; -import scala.Tuple2; - /** Submit Flink jobs to a YARN session cluster. */ -public final class YarnSessionClient extends YarnClientTrait { +public final class YarnSessionClient extends AbstractYarnClient { public static final YarnSessionClient INSTANCE = new YarnSessionClient(); @@ -62,25 +61,25 @@ private YarnSessionClient() { } @Override - public void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { + protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { super.setConfig(submitRequest, flinkConfig); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); logEffectiveSubmitConfiguration(flinkConfig); } - public void deployClusterConfig(DeployRequest deployRequest, Configuration flinkConfig) { + private void deployClusterConfig(DeployRequest deployRequest, Configuration flinkConfig) { ArrayList shipFiles = new ArrayList<>(); shipFiles.add(deployRequest.flinkVersion().getFlinkHome() + "/lib"); shipFiles.add(deployRequest.flinkVersion().getFlinkHome() + "/plugins"); - HdfsWorkspace hdfsWorkspace = deployRequest.hdfsWorkspace(); - FlinkConfigurationOps.safeSet( + RemoteWorkspace hdfsWorkspace = RemoteWorkspace.resolve(deployRequest.flinkVersion()); + FlinkConfigurationOps.setIfPresent( flinkConfig, YarnConfigOptions.FLINK_DIST_JAR, hdfsWorkspace.flinkDistJar()); - FlinkConfigurationOps.safeSet(flinkConfig, YarnConfigOptions.SHIP_FILES, shipFiles); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent(flinkConfig, YarnConfigOptions.SHIP_FILES, shipFiles); + FlinkConfigurationOps.setIfPresent( flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( flinkConfig, DeploymentOptionsInternal.CONF_DIR, deployRequest.flinkVersion().getFlinkHome() + "/conf"); @@ -89,51 +88,57 @@ public void deployClusterConfig(DeployRequest deployRequest, Configuration flink } @Override - public SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConfig) throws FlinkException { + protected SubmitResponse doSubmit( + SubmitRequest submitRequest, + Configuration flinkConfig) throws FlinkException { return callAsFlinkException( () -> { Tuple2 yarnClusterDescriptor = getYarnClusterDescriptor(flinkConfig); - ApplicationId yarnClusterId = yarnClusterDescriptor._1(); - YarnClusterDescriptor clusterDescriptor = yarnClusterDescriptor._2(); - - Tuple2 programJobGraph = - getJobGraph(flinkConfig, submitRequest, submitRequest.userJarFile()); - PackagedProgram packageProgram = programJobGraph._1(); - JobGraph jobGraph = programJobGraph._2(); + ApplicationId yarnClusterId = yarnClusterDescriptor._1; + YarnClusterDescriptor clusterDescriptor = yarnClusterDescriptor._2; - ClusterClient client = - clusterDescriptor.retrieve(yarnClusterId).getClusterClient(); - String jobId = client.submitJob(jobGraph).get().toString(); - String jobManagerUrl = client.getWebInterfaceURL(); + FlinkJobGraphBuilder.Result job = null; + ClusterClient client = null; + try { + job = + buildJobGraph( + flinkConfig, + submitRequest, + SubmitRequestResolver.userJarFile(submitRequest)); - logInfo( - String.format( - "%n-------------------------<>------------------------%n" - + "Flink Job Started: jobId: %s , applicationId: %s%n" - + "__________________________________________________________________%n", - jobId, yarnClusterId)); - - SubmitResponse resp = - new SubmitResponse( - yarnClusterId.toString(), flinkConfig.toMap(), jobId, jobManagerUrl); - closeSubmit(submitRequest, packageProgram, client, clusterDescriptor); - return resp; + client = clusterDescriptor.retrieve(yarnClusterId).getClusterClient(); + String jobId = client.submitJob(job.jobGraph()).get().toString(); + logInfo("Flink job started: jobId=" + jobId + ", applicationId=" + yarnClusterId); + return new SubmitResponse( + yarnClusterId.toString(), + flinkConfig.toMap(), + jobId, + client.getWebInterfaceURL()); + } finally { + closeSubmissionResources( + submitRequest, + job == null ? null : job.program(), + client, + clusterDescriptor); + } }); } @Override - public CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException { - FlinkConfigurationOps.safeSet( + protected CancelResponse doCancel( + CancelRequest cancelRequest, + Configuration flinkConfig) throws FlinkException { + FlinkConfigurationOps.setIfPresent( flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); return super.doCancel(cancelRequest, flinkConfig); } @Override - public SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest request, - Configuration flinkConfig) throws FlinkException { - FlinkConfigurationOps.safeSet( + protected SavepointResponse doTriggerSavepoint( + TriggerSavepointRequest request, + Configuration flinkConfig) throws FlinkException { + FlinkConfigurationOps.setIfPresent( flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); return super.doTriggerSavepoint(request, flinkConfig); } @@ -164,7 +169,7 @@ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { deployClusterConfig(deployRequest, flinkConfig); Tuple2 yarnClusterDescriptor = getYarnClusterDeployDescriptor(flinkConfig); - clusterDescriptor = yarnClusterDescriptor._2(); + clusterDescriptor = yarnClusterDescriptor._2; if (StringUtils.isNotBlank(deployRequest.clusterId())) { DeployResponse existingSession = @@ -176,7 +181,7 @@ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { client = clusterDescriptor - .deploySessionCluster(yarnClusterDescriptor._1()) + .deploySessionCluster(yarnClusterDescriptor._1) .getClusterClient(); if (client.getWebInterfaceURL() != null) { return new DeployResponse( @@ -224,55 +229,55 @@ private DeployResponse tryReuseExistingYarnSession( } } - public ShutDownResponse shutdown(ShutDownRequest shutDownRequest) throws Exception { + public ShutdownResponse shutdown(ShutdownRequest shutdownRequest) throws Exception { YarnClusterDescriptor clusterDescriptor = null; ClusterClient client = null; try { Configuration flinkConfig = - getFlinkDefaultConfiguration(shutDownRequest.flinkVersion().getFlinkHome()); - if (shutDownRequest.properties() != null) { - for (Map.Entry entry : shutDownRequest.properties().entrySet()) { + loadDefaultConfiguration(shutdownRequest.flinkVersion().getFlinkHome()); + if (shutdownRequest.properties() != null) { + for (Map.Entry entry : shutdownRequest.properties().entrySet()) { if (entry.getValue() != null) { flinkConfig.setString(entry.getKey(), entry.getValue().toString()); } } } - FlinkConfigurationOps.safeSet( - flinkConfig, YarnConfigOptions.APPLICATION_ID, shutDownRequest.clusterId()); - FlinkConfigurationOps.safeSet( + FlinkConfigurationOps.setIfPresent( + flinkConfig, YarnConfigOptions.APPLICATION_ID, shutdownRequest.clusterId()); + FlinkConfigurationOps.setIfPresent( flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); Tuple2 yarnClusterDescriptor = getYarnClusterDescriptor(flinkConfig); - clusterDescriptor = yarnClusterDescriptor._2(); + clusterDescriptor = yarnClusterDescriptor._2; - boolean shutDownState = + boolean shutdownState = FinalApplicationStatus.UNDEFINED.equals( clusterDescriptor .getYarnClient() .getApplicationReport( - ApplicationId.fromString(shutDownRequest.clusterId())) + ApplicationId.fromString(shutdownRequest.clusterId())) .getFinalApplicationStatus()); - if (shutDownState) { + if (shutdownState) { client = clusterDescriptor - .retrieve(yarnClusterDescriptor._1()) + .retrieve(yarnClusterDescriptor._1) .getClusterClient(); client.shutDownCluster(); } logInfo( "the " - + shutDownRequest.clusterId() + + shutdownRequest.clusterId() + "'s final status is " + clusterDescriptor .getYarnClient() .getApplicationReport( - ConverterUtils.toApplicationId(shutDownRequest.clusterId())) + ConverterUtils.toApplicationId(shutdownRequest.clusterId())) .getFinalApplicationStatus()); - return new ShutDownResponse(shutDownRequest.clusterId()); + return new ShutdownResponse(shutdownRequest.clusterId()); } catch (Exception e) { - logError("shutdown flink session fail in " + shutDownRequest.deployMode() + " mode"); + logError("shutdown flink session fail in " + shutdownRequest.deployMode() + " mode"); throw e; } finally { Utils.close(client, clusterDescriptor); diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/tool/FlinkSessionSubmitHelper.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/tool/FlinkSessionSubmitHelper.java deleted file mode 100644 index 82624b460b..0000000000 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/tool/FlinkSessionSubmitHelper.java +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.client.tool; - -import org.apache.streampark.common.util.AssertUtils; -import org.apache.streampark.common.util.JsonUtils; -import org.apache.streampark.common.util.LoggerSupport; -import org.apache.streampark.flink.client.conf.FlinkSavepointOptions; - -import org.apache.streampark.shaded.com.fasterxml.jackson.databind.JsonNode; - -import org.apache.flink.client.deployment.application.ApplicationConfiguration; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.CoreOptions; - -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.security.AccessController; -import java.security.PrivilegedExceptionAction; -import java.time.Duration; -import java.util.List; - -/** Submit Flink jobs to session clusters via the REST API. */ -public final class FlinkSessionSubmitHelper extends LoggerSupport { - - private static final FlinkSessionSubmitHelper INSTANCE = new FlinkSessionSubmitHelper(); - - private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(30); - - private static final Duration REQUEST_TIMEOUT = Duration.ofMinutes(5); - - private FlinkSessionSubmitHelper() { - } - - /** - * Submit Flink Job via Rest API. - * - * @param jmRestUrl jobmanager rest url of target flink cluster - * @param flinkJobJar flink job jar file - * @param flinkConfig flink configuration - * @return jobID of submitted flink job - */ - public static String submitViaRestApi( - String jmRestUrl, File flinkJobJar, - Configuration flinkConfig) throws Exception { - return INSTANCE.doSubmitViaRestApi(jmRestUrl, flinkJobJar, flinkConfig); - } - - private String doSubmitViaRestApi(String jmRestUrl, File flinkJobJar, Configuration flinkConfig) throws Exception { - HttpClient httpClient = - HttpClient.newBuilder().connectTimeout(CONNECT_TIMEOUT).build(); - - String boundary = "----StreamParkBoundary" + System.currentTimeMillis(); - HttpRequest uploadRequest = - HttpRequest.newBuilder() - .uri(URI.create(jmRestUrl + "/jars/upload")) - .timeout(REQUEST_TIMEOUT) - .header("Content-Type", "multipart/form-data; boundary=" + boundary) - .POST(HttpRequest.BodyPublishers.ofByteArray(buildMultipartBody(boundary, flinkJobJar))) - .build(); - - String uploadResult = - AccessController.doPrivileged( - (PrivilegedExceptionAction) () -> httpClient - .send( - uploadRequest, - HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)) - .body()); - - JarUploadResponse jarUploadResponse = parseJarUploadResponse(uploadResult); - - AssertUtils.required( - jarUploadResponse != null && jarUploadResponse.isSuccessful(), - "[flink-submit] upload flink jar to flink session cluster failed, jmRestUrl=" - + jmRestUrl - + ", response=" - + jarUploadResponse); - - HttpRequest runRequest = - HttpRequest.newBuilder() - .uri(URI.create(jmRestUrl + "/jars/" + jarUploadResponse.jarId() + "/run")) - .timeout(REQUEST_TIMEOUT) - .header("Content-Type", "application/json") - .POST(HttpRequest.BodyPublishers.ofString(JsonUtils.write(new JarRunRequest(flinkConfig)))) - .build(); - - String resp = - AccessController.doPrivileged( - (PrivilegedExceptionAction) () -> httpClient - .send( - runRequest, - HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8)) - .body()); - - String jobId = parseJobId(resp); - if (jobId == null || jobId.isBlank()) { - throw new IllegalStateException( - "[flink-submit] submit flink job via rest api failed, jmRestUrl=" - + jmRestUrl - + ", response=" - + resp); - } - return jobId; - } - - private byte[] buildMultipartBody(String boundary, File jarFile) throws IOException { - String header = - "--" - + boundary - + "\r\n" - + "Content-Disposition: form-data; name=\"jarfile\"; filename=\"" - + jarFile.getName() - + "\"\r\n" - + "Content-Type: application/java-archive\r\n\r\n"; - byte[] headerBytes = header.getBytes(StandardCharsets.UTF_8); - byte[] fileBytes = Files.readAllBytes(jarFile.toPath()); - byte[] footerBytes = ("\r\n--" + boundary + "--\r\n").getBytes(StandardCharsets.UTF_8); - byte[] body = new byte[headerBytes.length + fileBytes.length + footerBytes.length]; - System.arraycopy(headerBytes, 0, body, 0, headerBytes.length); - System.arraycopy(fileBytes, 0, body, headerBytes.length, fileBytes.length); - System.arraycopy(footerBytes, 0, body, headerBytes.length + fileBytes.length, footerBytes.length); - return body; - } - - private JarUploadResponse parseJarUploadResponse(String uploadResult) { - try { - JsonNode node = JsonUtils.read(uploadResult, JsonNode.class); - return new JarUploadResponse( - node.has("filename") ? node.get("filename").asText(null) : null, - node.has("status") ? node.get("status").asText(null) : null); - } catch (Exception e) { - return null; - } - } - - private String parseJobId(String resp) { - try { - JsonNode node = JsonUtils.read(resp, JsonNode.class); - if (node.has("errors") && node.get("errors").size() > 0) { - return null; - } - if (node.has("jobid")) { - return node.get("jobid").asText(null); - } - if (node.has("jobId")) { - return node.get("jobId").asText(null); - } - return null; - } catch (Exception e) { - return null; - } - } -} - -/** - * refer to https://ci.apache.org/projects/flink/flink-docs-stable/docs/ops/rest_api/#jars-upload - */ -class JarUploadResponse { - - private final String filename; - private final String status; - - JarUploadResponse(String filename, String status) { - this.filename = filename; - this.status = status; - } - - boolean isSuccessful() { - return "success".equalsIgnoreCase(status); - } - - String jarId() { - return filename.substring(filename.lastIndexOf('/') + 1); - } - - @Override - public String toString() { - return "JarUploadResponse{filename='" + filename + "', status='" + status + "'}"; - } -} - -/** - * refer to https://ci.apache.org/projects/flink/flink-docs-stable/docs/ops/rest_api/#jars-upload - */ -class JarRunRequest { - - private final String entryClass; - private final String programArgs; - private final String parallelism; - private final String savepointPath; - private final boolean allowNonRestoredState; - - JarRunRequest(Configuration flinkConf) { - this.entryClass = flinkConf.get(ApplicationConfiguration.APPLICATION_MAIN_CLASS); - List args = flinkConf.get(ApplicationConfiguration.APPLICATION_ARGS); - this.programArgs = args == null ? null : String.join(" ", args); - this.parallelism = String.valueOf(flinkConf.get(CoreOptions.DEFAULT_PARALLELISM)); - this.savepointPath = flinkConf.get(FlinkSavepointOptions.SAVEPOINT_PATH); - this.allowNonRestoredState = - flinkConf.get(FlinkSavepointOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE); - } - - public String getEntryClass() { - return entryClass; - } - - public String getProgramArgs() { - return programArgs; - } - - public String getParallelism() { - return parallelism; - } - - public String getSavepointPath() { - return savepointPath; - } - - public boolean isAllowNonRestoredState() { - return allowNonRestoredState; - } -} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/FlinkClientTrait.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/FlinkClientTrait.java deleted file mode 100644 index 5243a034c2..0000000000 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/trait/FlinkClientTrait.java +++ /dev/null @@ -1,1042 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.client.trait; - -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.conf.Workspace; -import org.apache.streampark.common.constants.Constants; -import org.apache.streampark.common.enums.ApplicationType; -import org.apache.streampark.common.enums.FlinkDeployMode; -import org.apache.streampark.common.enums.FlinkJobType; -import org.apache.streampark.common.enums.FlinkRestoreMode; -import org.apache.streampark.common.fs.FsOperator; -import org.apache.streampark.common.util.AssertUtils; -import org.apache.streampark.common.util.ClassLoaderUtils; -import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.ExceptionUtils; -import org.apache.streampark.common.util.FlinkConfigurationUtils; -import org.apache.streampark.common.util.LoggerSupport; -import org.apache.streampark.common.util.SystemPropertyUtils; -import org.apache.streampark.common.util.Utils; -import org.apache.streampark.flink.client.bean.CancelRequest; -import org.apache.streampark.flink.client.bean.CancelResponse; -import org.apache.streampark.flink.client.bean.SavepointRequestTrait; -import org.apache.streampark.flink.client.bean.SavepointResponse; -import org.apache.streampark.flink.client.bean.SubmitRequest; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.client.bean.TriggerSavepointRequest; -import org.apache.streampark.flink.client.conf.FlinkSavepointOptions; -import org.apache.streampark.flink.core.FlinkClusterClient; -import org.apache.streampark.flink.core.conf.FlinkRunOption; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Options; -import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.common.JobID; -import org.apache.flink.client.cli.CliArgsException; -import org.apache.flink.client.cli.CliFrontend; -import org.apache.flink.client.cli.CliFrontendParser; -import org.apache.flink.client.cli.CustomCommandLine; -import org.apache.flink.client.cli.ExecutionConfigAccessor; -import org.apache.flink.client.cli.ProgramOptions; -import org.apache.flink.client.deployment.application.ApplicationConfiguration; -import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.PackagedProgram; -import org.apache.flink.client.program.PackagedProgramUtils; -import org.apache.flink.configuration.CheckpointingOptions; -import org.apache.flink.configuration.ConfigConstants; -import org.apache.flink.configuration.ConfigOption; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.CoreOptions; -import org.apache.flink.configuration.DeploymentOptions; -import org.apache.flink.configuration.ExecutionOptions; -import org.apache.flink.configuration.GlobalConfiguration; -import org.apache.flink.configuration.PipelineOptions; -import org.apache.flink.configuration.PipelineOptionsInternal; -import org.apache.flink.python.PythonOptions; -import org.apache.flink.runtime.jobgraph.JobGraph; -import org.apache.flink.util.FlinkException; -import org.apache.flink.util.Preconditions; - -import com.google.common.collect.Lists; - -import java.io.File; -import java.net.URI; -import java.net.URL; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import scala.Tuple2; - -/** Base trait for Flink client implementations. */ -public abstract class FlinkClientTrait extends LoggerSupport { - - private final String paramKeyFlinkConf = - ConfigKeys.KEY_FLINK_CONF(ConfigKeys.PARAM_PREFIX()); - private final String paramKeyFlinkSql = - ConfigKeys.KEY_FLINK_SQL(ConfigKeys.PARAM_PREFIX()); - private final String paramKeyAppConf = - ConfigKeys.KEY_APP_CONF(ConfigKeys.PARAM_PREFIX()); - private final String paramKeyAppName = - ConfigKeys.KEY_APP_NAME(ConfigKeys.PARAM_PREFIX()); - private final String paramKeyFlinkParallelism = - ConfigKeys.KEY_FLINK_PARALLELISM(ConfigKeys.PARAM_PREFIX()); - - protected static FlinkException asFlinkException(Throwable throwable) { - if (throwable instanceof FlinkException) { - return (FlinkException) throwable; - } - if (throwable instanceof Exception) { - return new FlinkException((Exception) throwable); - } - return new FlinkException(throwable.getMessage(), throwable); - } - - @FunctionalInterface - protected interface FlinkCallable { - - T call() throws Exception; - } - - protected static T callAsFlinkException(FlinkCallable callable) throws FlinkException { - return callAsFlinkException(callable, e -> { - }); - } - - protected static T callAsFlinkException( - FlinkCallable callable, - java.util.function.Consumer onFailure) throws FlinkException { - try { - return callable.call(); - } catch (FlinkException e) { - onFailure.accept(e); - throw e; - } catch (Exception e) { - onFailure.accept(e); - throw asFlinkException(e); - } - } - - protected static T callAsFlinkExceptionMapping( - FlinkCallable callable, - java.util.function.Function exceptionMapper) throws FlinkException { - try { - return callable.call(); - } catch (FlinkException e) { - throw e; - } catch (Exception e) { - throw exceptionMapper.apply(e); - } - } - - protected CancelResponse toCancelResponse( - CancelRequest request, JobID jobId, - ClusterClient client) throws FlinkException { - return callAsFlinkException(() -> new CancelResponse(cancelJob(request, jobId, client))); - } - - protected SavepointResponse toSavepointResponse( - TriggerSavepointRequest request, - JobID jobId, - ClusterClient client) throws FlinkException { - return callAsFlinkException( - () -> new SavepointResponse(triggerSavepoint(request, jobId, client))); - } - - protected void logEffectiveSubmitConfiguration(Configuration flinkConfig) { - logInfo( - String.format( - "%n------------------------------------------------------------------%n" - + "Effective submit configuration: %s%n" - + "------------------------------------------------------------------%n", - flinkConfig)); - } - - protected void logSavepointClientRequest(String operation, SavepointRequestTrait request) { - StringBuilder message = - new StringBuilder() - .append("\n----------------------------------------- flink job ") - .append(operation) - .append(" --------------------------------\n") - .append(" userFlinkHome : ") - .append(request.flinkVersion().getFlinkHome()) - .append("\n") - .append(" flinkVersion : ") - .append(request.flinkVersion().version()) - .append("\n") - .append(" clusterId : ") - .append(request.clusterId()) - .append("\n"); - if (request instanceof CancelRequest) { - CancelRequest cancelRequest = (CancelRequest) request; - message - .append(" withSavePoint : ") - .append(cancelRequest.withSavepoint()) - .append("\n"); - } - message - .append(" savePointPath : ") - .append(request.savepointPath()) - .append("\n"); - if (request instanceof CancelRequest) { - message - .append(" withDrain : ") - .append(((CancelRequest) request).withDrain()) - .append("\n"); - } - message - .append(" nativeFormat : ") - .append(request.nativeFormat()) - .append("\n") - .append(" k8sNamespace : ") - .append(request.kubernetesNamespace()) - .append("\n") - .append(" appId : ") - .append(request.clusterId()) - .append("\n") - .append(" jobId : ") - .append(request.jobId()) - .append("\n") - .append( - "-------------------------------------------------------------------------------------------\n"); - logInfo(message.toString()); - } - - /** - * Submits a job graph, tolerating the signature change {@code ClusterClient#submitJob} went - * through: it took a {@code JobGraph} until Flink 2.x widened the parameter to {@code - * ExecutionPlan}, which {@code JobGraph} implements. The instance is accepted by either - * version — only the declared parameter type moved — but this module is compiled once against - * a single baseline, so a direct call binds to one signature and fails against the other with - * {@code NoSuchMethodError}. - */ - private static String submitJobGraph(ClusterClient client, JobGraph jobGraph) throws Exception { - for (java.lang.reflect.Method method : client.getClass().getMethods()) { - if ("submitJob".equals(method.getName()) - && method.getParameterCount() == 1 - && method.getParameterTypes()[0].isInstance(jobGraph)) { - Object future = method.invoke(client, jobGraph); - return ((java.util.concurrent.CompletableFuture) future).get().toString(); - } - } - throw new FlinkException( - "No ClusterClient#submitJob(..) accepting a JobGraph on " + client.getClass().getName()); - } - - protected SubmitResponse submitJobGraphToCluster( - SubmitRequest submitRequest, - Configuration flinkConfig, - File jarFile, - FlinkCallable> clientSupplier, - FlinkCallable clusterIdSupplier, - AutoCloseable... extraResources) throws FlinkException { - return callAsFlinkException( - () -> { - Tuple2 programJobGraph = - getJobGraph(flinkConfig, submitRequest, jarFile); - PackagedProgram packageProgram = programJobGraph._1(); - JobGraph jobGraph = programJobGraph._2(); - ClusterClient client = clientSupplier.call(); - String jobId = submitJobGraph(client, jobGraph); - SubmitResponse result = - new SubmitResponse( - clusterIdSupplier.call(), - flinkConfig.toMap(), - jobId, - client.getWebInterfaceURL()); - AutoCloseable[] resources = new AutoCloseable[extraResources.length + 2]; - resources[0] = packageProgram; - resources[1] = client; - System.arraycopy(extraResources, 0, resources, 2, extraResources.length); - closeSubmit(submitRequest, resources); - return result; - }); - } - - public SubmitResponse submit(SubmitRequest submitRequest) throws FlinkException { - logInfo( - "\n" - + "--------------------------------------- flink job start ---------------------------------------\n" - + " userFlinkHome : " - + submitRequest.flinkVersion().getFlinkHome() - + "\n" - + " flinkVersion : " - + submitRequest.flinkVersion().version() - + "\n" - + " appName : " - + submitRequest.effectiveAppName() - + "\n" - + " jobType : " - + submitRequest.jobType().name() - + "\n" - + " deployMode : " - + submitRequest.deployMode().name() - + "\n" - + " k8sNamespace : " - + submitRequest.kubernetesNamespace() - + "\n" - + " flinkExposedType : " - + submitRequest.flinkRestExposedType() - + "\n" - + " clusterId : " - + submitRequest.clusterId() - + "\n" - + " applicationType : " - + submitRequest.applicationType().getName() - + "\n" - + " savePoint : " - + submitRequest.savePoint() - + "\n" - + " properties : " - + formatProperties(submitRequest.properties()) - + "\n" - + " args : " - + submitRequest.args() - + "\n" - + " appConf : " - + submitRequest.appConf() - + "\n" - + " flinkBuildResult : " - + submitRequest.buildResult() - + "\n" - + "-------------------------------------------------------------------------------------------\n"); - - Configuration flinkConfig = callAsFlinkException(() -> prepareConfig(submitRequest)); - setConfig(submitRequest, flinkConfig); - - return callAsFlinkException( - () -> doSubmit(submitRequest, flinkConfig), - e -> { - logSubmitFailure(submitRequest, e); - }); - } - - private void logSubmitFailure(SubmitRequest submitRequest, Exception e) { - logError( - "flink job " - + submitRequest.appName() - + " start failed, " - + "deployMode: " - + submitRequest.deployMode().getName() - + ", " - + "detail: " - + ExceptionUtils.stringifyException(e)); - } - - private Configuration prepareConfig(SubmitRequest submitRequest) throws Exception { - CommandLineAndConfig commandLineAndConfig = getCommandLineAndFlinkConfig(submitRequest); - Configuration flinkConfig = commandLineAndConfig.flinkConfig; - applyJobTypeConfig(submitRequest, commandLineAndConfig.commandLine, flinkConfig); - applyCommonPipelineConfig(submitRequest, flinkConfig); - applyCheckpointDefaults(submitRequest, flinkConfig); - applySavepointConfig(submitRequest, flinkConfig); - applyEnvProperties(submitRequest, flinkConfig); - return flinkConfig; - } - - private void applyJobTypeConfig( - SubmitRequest submitRequest, CommandLine commandLine, - Configuration flinkConfig) throws Exception { - if (submitRequest.jobType() == FlinkJobType.PYFLINK) { - applyPyFlinkConfig(submitRequest, flinkConfig); - return; - } - if (submitRequest.userJarFile() != null) { - URI uri = - PackagedProgramUtils.resolveURI( - submitRequest.userJarFile().getAbsolutePath()); - ProgramOptions programOptions = ProgramOptions.create(commandLine); - ExecutionConfigAccessor executionParameters = - ExecutionConfigAccessor.fromProgramOptions( - programOptions, Collections.singletonList(uri.toString())); - executionParameters.applyToConfiguration(flinkConfig); - } - } - - private void applyPyFlinkConfig(SubmitRequest submitRequest, Configuration flinkConfig) throws Exception { - String pythonVenv = Workspace.local().APP_PYTHON_VENV(); - AssertUtils.required( - FsOperator.lfs().exists(pythonVenv), - pythonVenv + " File does not exist"); - - safeSet(flinkConfig, PythonOptions.PYTHON_ARCHIVES, pythonVenv); - safeSet( - flinkConfig, - PythonOptions.PYTHON_CLIENT_EXECUTABLE, - Constants.PYTHON_EXECUTABLE); - safeSet(flinkConfig, PythonOptions.PYTHON_EXECUTABLE, Constants.PYTHON_EXECUTABLE); - - String flinkOptPath = System.getenv(ConfigConstants.ENV_FLINK_OPT_DIR); - if (StringUtils.isBlank(flinkOptPath)) { - logWarn( - "Get environment variable " - + ConfigConstants.ENV_FLINK_OPT_DIR - + " fail"); - String flinkHome = submitRequest.flinkVersion().getFlinkHome(); - SystemPropertyUtils.setEnv( - ConfigConstants.ENV_FLINK_OPT_DIR, flinkHome + "/opt"); - logInfo( - "Set temporary environment variables " - + ConfigConstants.ENV_FLINK_OPT_DIR - + " = " - + flinkHome - + "/opt"); - } - } - - private void applyCommonPipelineConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - safeSet(flinkConfig, PipelineOptions.NAME, submitRequest.effectiveAppName()); - safeSet(flinkConfig, DeploymentOptions.TARGET, submitRequest.deployMode().getName()); - safeSet(flinkConfig, FlinkSavepointOptions.SAVEPOINT_PATH, submitRequest.savePoint()); - safeSet( - flinkConfig, - ApplicationConfiguration.APPLICATION_MAIN_CLASS, - submitRequest.appMain()); - safeSet( - flinkConfig, - ApplicationConfiguration.APPLICATION_ARGS, - extractProgramArgs(submitRequest)); - safeSet( - flinkConfig, - PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, - submitRequest.jobId()); - } - - private void applyCheckpointDefaults(SubmitRequest submitRequest, Configuration flinkConfig) throws Exception { - if (!submitRequest.hasProp(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.key())) { - Configuration flinkDefaultConfiguration = - getFlinkDefaultConfiguration(submitRequest.flinkVersion().getFlinkHome()); - ConfigOption retainedOption = - CheckpointingOptions.MAX_RETAINED_CHECKPOINTS; - flinkConfig.set( - retainedOption, flinkDefaultConfiguration.get(retainedOption)); - } - } - - private void applySavepointConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - if (StringUtils.isBlank(submitRequest.savePoint())) { - return; - } - safeSet( - flinkConfig, - FlinkSavepointOptions.SAVEPOINT_PATH, - submitRequest.savePoint()); - flinkConfig.set( - FlinkSavepointOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE, - submitRequest.allowNonRestoredState()); - boolean enableRestoreMode = - submitRequest.restoreMode() != null - && submitRequest - .flinkVersion() - .checkVersion(FlinkRestoreMode.SINCE_FLINK_VERSION); - if (enableRestoreMode) { - flinkConfig.setString( - FlinkRestoreMode.RESTORE_MODE, submitRequest.restoreMode().getName()); - } - } - - private void applyEnvProperties(SubmitRequest submitRequest, Configuration flinkConfig) { - if (MapUtils.isEmpty(submitRequest.properties())) { - return; - } - if (submitRequest.hasProp(CoreOptions.FLINK_JVM_OPTIONS.key())) { - Object jvmOptObj = - submitRequest.getProp(CoreOptions.FLINK_JVM_OPTIONS.key()); - String jvmOpt = jvmOptObj.toString(); - if (!jvmOpt.contains("-Dfile.encoding=")) { - submitRequest.putProperty( - CoreOptions.FLINK_JVM_OPTIONS.key(), "-Dfile.encoding=UTF-8 " + jvmOpt); - } - } - - for (Map.Entry entry : submitRequest.properties().entrySet()) { - if (entry.getKey().startsWith("env.")) { - logInfo("env opts: " + entry.getKey() + ": " + entry.getValue()); - flinkConfig.setString(entry.getKey(), entry.getValue().toString()); - } - } - } - - public abstract void setConfig(SubmitRequest submitRequest, Configuration flinkConf); - - public SavepointResponse triggerSavepoint(TriggerSavepointRequest savepointRequest) throws FlinkException { - logSavepointClientRequest("trigger savepoint", savepointRequest); - Configuration flinkConf = new Configuration(); - return callAsFlinkException(() -> doTriggerSavepoint(savepointRequest, flinkConf)); - } - - public CancelResponse cancel(CancelRequest cancelRequest) throws FlinkException { - logSavepointClientRequest("cancel", cancelRequest); - Configuration flinkConf = new Configuration(); - return callAsFlinkException(() -> doCancel(cancelRequest, flinkConf)); - } - - public abstract SubmitResponse doSubmit(SubmitRequest submitRequest, Configuration flinkConf) throws FlinkException; - - public abstract SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest request, - Configuration flinkConf) throws FlinkException; - - public abstract CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConf) throws FlinkException; - - protected SubmitResponse trySubmit( - SubmitRequest submitRequest, - Configuration flinkConfig, - File jarFile, - SubmitFunc jobGraphFunc, - SubmitFunc restApiFunc) throws FlinkException { - try { - logInfo("[flink-submit] Submit job with JobGraph Plan."); - return jobGraphFunc.apply(submitRequest, flinkConfig, jarFile); - } catch (FlinkException e) { - logWarn( - "[flink-submit] JobGraph submit plan failed, falling back to Rest API submit plan: " - + ExceptionUtils.stringifyException(e)); - try { - return restApiFunc.apply(submitRequest, flinkConfig, jarFile); - } catch (FlinkException fallbackException) { - throw new FlinkException( - "[flink-submit] Both JobGraph submit plan and Rest API submit plan all failed!\n" - + "JobGraph Submit plan failed detail:\n" - + "------------------------------------------------------------------\n" - + ExceptionUtils.stringifyException(e) - + "\n" - + "------------------------------------------------------------------\n" - + "\n" - + " RestAPI Submit plan failed detail:\n" - + " ------------------------------------------------------------------\n" - + ExceptionUtils.stringifyException(fallbackException) - + "\n" - + "------------------------------------------------------------------\n", - fallbackException); - } - } - } - - /** - * Makes the program's classloader resolve {@code org.apache.streampark.*} from its parent — the - * shims classloader built for the registered Flink version — instead of from the submitted jar. - * - *

Set through the raw key rather than {@code CoreOptions}: the typed accessors around it - * moved between Flink 1.x and 2.x, while the key itself did not, and this module is compiled - * against a single baseline but submits to whichever version the user registered. - */ - private static Configuration streamParkParentFirstConfig() { - Configuration configuration = new Configuration(); - configuration.setString( - "classloader.parent-first-patterns.additional", "org.apache.streampark."); - return configuration; - } - - /** - * Parent-first patterns for FLINK_SQL fat jars, which bundle a SQL client but not the target - * Flink distribution. Without delegating {@code org.apache.flink.*} and {@code org.yaml.*} - * to the registered Flink version's lib directory, shaded YAML/Jackson classes from the jar - * collide with Flink's own copies and fail with {@code LinkageError}. - */ - private static Configuration flinkSqlParentFirstConfig() { - Configuration configuration = new Configuration(); - configuration.setString( - "classloader.parent-first-patterns.additional", - "org.apache.streampark.;org.apache.flink.;org.yaml."); - return configuration; - } - - /** - * Parent for {@link org.apache.flink.client.program.PackagedProgram}'s user-code classloader. - * - *

Flink's parent-first patterns delegate matching classes to this loader before the SQL fat - * jar. The fat jar bundles an unshaded {@code org.yaml.snakeyaml} copy that clashes with Flink - * 2.x's shaded YAML stack, and it omits {@code org.apache.flink.*} entirely. StreamPark shims - * carry the registered Flink version's classes; the console classpath carries snakeyaml. - */ - private static ClassLoader packagedProgramParentClassLoader(ClassLoader shimsClassLoader) { - ClassLoader consoleClassLoader = FlinkClientTrait.class.getClassLoader(); - return new ClassLoader(shimsClassLoader) { - - @Override - protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { - synchronized (getClassLoadingLock(name)) { - Class loaded = findLoadedClass(name); - if (loaded == null) { - if (name.startsWith("org.apache.streampark.") - || name.startsWith("org.apache.flink.")) { - loaded = getParent().loadClass(name); - } else if (name.startsWith("org.yaml.")) { - loaded = consoleClassLoader.loadClass(name); - } - } - if (loaded == null) { - loaded = super.loadClass(name, resolve); - } else if (resolve) { - resolveClass(loaded); - } - return loaded; - } - } - }; - } - - private static List jobGraphUserClassPaths(SubmitRequest submitRequest) { - return jobGraphUserClassPaths(submitRequest, false); - } - - /** - * Classpath entries for {@link PackagedProgram} when building a JobGraph locally. - * - * @param includeFlinkDist keep {@code flink-dist} on the classpath; required for thin uploaded - * JARs on Flink 1.20+ where runtime classes live in {@code flink-dist} rather than {@code - * flink-core}. FLINK_SQL fat jars omit it to avoid snakeyaml clashes and rely on parent-first - * delegation instead. - */ - private static List jobGraphUserClassPaths( - SubmitRequest submitRequest, boolean includeFlinkDist) { - List classPaths = new ArrayList<>(submitRequest.classPaths()); - classPaths.removeIf( - url -> { - String name = url.getPath(); - if (name.contains("flink-shaded-force-shading")) { - return true; - } - return !includeFlinkDist && name.contains("flink-dist"); - }); - return classPaths; - } - - public Tuple2 getJobGraph( - Configuration flinkConfig, SubmitRequest submitRequest, - File jarFile) throws Exception { - PackagedProgram.Builder builder = - PackagedProgram.newBuilder() - .setSavepointRestoreSettings(submitRequest.savepointRestoreSettings()) - .setEntryPointClassName( - flinkConfig - .getOptional(ApplicationConfiguration.APPLICATION_MAIN_CLASS) - .orElseThrow( - () -> new IllegalStateException( - "Application main class is not configured"))) - .setArguments( - flinkConfig - .getOptional(ApplicationConfiguration.APPLICATION_ARGS) - .orElse(Lists.newArrayList()) - .toArray(new String[0])); - - if (submitRequest.jobType() == FlinkJobType.PYFLINK) { - if (!submitRequest.libs().isEmpty()) { - // BUG: https://github.com/apache/streampark/issues/3761 - // builder.setUserClassPaths(Lists.newArrayList(submitRequest.libs())) - } - } else { - builder.setJarFile(jarFile); - boolean flinkSqlJob = submitRequest.jobType() == FlinkJobType.FLINK_SQL; - // Thin user JARs (Flink examples, uploaded jobs) and FLINK_SQL fat jars both rely on the - // registered Flink version's lib/ at JobGraph build time. Parent-first delegation keeps - // org.apache.streampark.* on the shims classloader for SQL; uploaded JARs also need - // flink-dist when the target Flink version bundles runtime classes there (1.20+). - builder.setUserClassPaths( - jobGraphUserClassPaths(submitRequest, !flinkSqlJob)); - builder.setConfiguration(flinkSqlParentFirstConfig()); - } - - ClassLoader shimsClassLoader = Thread.currentThread().getContextClassLoader(); - ClassLoader programParent = - submitRequest.jobType() == FlinkJobType.PYFLINK - ? shimsClassLoader - : packagedProgramParentClassLoader(shimsClassLoader); - Thread.currentThread().setContextClassLoader(programParent); - try { - PackagedProgram packageProgram = builder.build(); - Configuration jobGraphConfig = new Configuration(flinkConfig); - if (submitRequest.jobType() != FlinkJobType.PYFLINK) { - jobGraphConfig.addAll(flinkSqlParentFirstConfig()); - } - JobGraph jobGraph = - PackagedProgramUtils.createJobGraph( - packageProgram, - jobGraphConfig, - getParallelism(submitRequest), - null, - false); - - return new Tuple2<>(packageProgram, jobGraph); - } finally { - Thread.currentThread().setContextClassLoader(shimsClassLoader); - } - } - - public JobID getJobID(String jobId) throws CliArgsException { - try { - return JobID.fromHexString(jobId); - } catch (Exception e) { - throw new CliArgsException(e.getMessage()); - } - } - - CustomCommandLine validateAndGetActiveCommandLine( - List customCommandLines, - CommandLine commandLine) { - CommandLine line = Preconditions.checkNotNull(commandLine); - logInfo("Custom commandline: " + customCommandLines); - for (CustomCommandLine cli : customCommandLines) { - boolean isActive = cli.isActive(line); - logInfo("Checking custom commandline " + cli + ", isActive: " + isActive); - if (isActive) { - return cli; - } - } - throw new IllegalStateException("No valid command-line found."); - } - - public Configuration getFlinkDefaultConfiguration(String flinkHome) { - try { - return GlobalConfiguration.loadConfiguration(flinkHome + "/conf"); - } catch (Exception e) { - return new Configuration(); - } - } - - T getOptionFromDefaultFlinkConfig(String flinkHome, ConfigOption option) { - return getFlinkDefaultConfiguration(flinkHome).get(option); - } - - List getCustomCommandLines(String flinkHome) { - Configuration flinkDefaultConfiguration = getFlinkDefaultConfiguration(flinkHome); - String confDir = flinkHome + "/conf"; - // CliFrontend/GenericCLI are bound to the Flink version bundled with this module (loaded by - // this class's own classloader), but the calling thread's context classloader may currently - // be a target-version shims classloader (see FlinkShimsProxy). GenericCLI's internal - // ServiceLoader.load(PipelineExecutorFactory.class) resolves providers via the context - // classloader, so leaving it as the shims classloader here would load a PipelineExecutorFactory - // implementation from a different Flink version than the interface bundled here, throwing - // ServiceConfigurationError ("not a subtype"). Force it back to this class's own classloader - // for the duration of this call. - return ClassLoaderUtils.runAsClassLoader( - FlinkClientTrait.class.getClassLoader(), - () -> CliFrontend.loadCustomCommandLines(flinkDefaultConfiguration, confDir)); - } - - public Integer getParallelism(SubmitRequest submitRequest) { - if (submitRequest.hasProp(ConfigKeys.KEY_FLINK_PARALLELISM())) { - return Integer.valueOf( - submitRequest.getProp(ConfigKeys.KEY_FLINK_PARALLELISM()).toString()); - } - return getFlinkDefaultConfiguration(submitRequest.flinkVersion().getFlinkHome()) - .get(CoreOptions.DEFAULT_PARALLELISM, CoreOptions.DEFAULT_PARALLELISM.defaultValue()); - } - - Options getCommandLineOptions(String flinkHome) { - List customCommandLines = getCustomCommandLines(flinkHome); - Options customCommandLineOptions = new Options(); - for (CustomCommandLine customCommandLine : customCommandLines) { - customCommandLine.addGeneralOptions(customCommandLineOptions); - customCommandLine.addRunOptions(customCommandLineOptions); - } - return FlinkRunOption.mergeOptions( - CliFrontendParser.getRunCommandOptions(), customCommandLineOptions); - } - - public Configuration extractConfiguration(String flinkHome, Map properties) throws Exception { - Options commandLineOptions = getCommandLineOptions(flinkHome); - List cliArgs = new ArrayList<>(); - if (MapUtils.isNotEmpty(properties)) { - for (Map.Entry entry : properties.entrySet()) { - cliArgs.add("-D" + entry.getKey() + "=" + entry.getValue().toString().trim()); - } - } - CommandLine commandLine = - FlinkRunOption.parse( - commandLineOptions, cliArgs.toArray(new String[0]), true); - CustomCommandLine activeCommandLine = - validateAndGetActiveCommandLine(getCustomCommandLines(flinkHome), commandLine); - return applyConfiguration(flinkHome, activeCommandLine, commandLine); - } - - public String cancelJob(CancelRequest cancelRequest, JobID jobID, ClusterClient client) throws Exception { - String savePointDir = tryGetSavepointPathIfNeed(cancelRequest); - - FlinkClusterClient clientWrapper = new FlinkClusterClient<>(client); - boolean withSavepoint = cancelRequest.withSavepoint(); - boolean withDrain = cancelRequest.withDrain(); - - if (!withSavepoint && !withDrain) { - client.cancel(jobID).get(); - return null; - } - return clientWrapper - .stopWithSavepoint( - jobID, cancelRequest.withDrain(), savePointDir, cancelRequest.nativeFormat()) - .get(); - } - - public String triggerSavepoint( - TriggerSavepointRequest savepointRequest, JobID jobID, - ClusterClient client) throws Exception { - String savepointPath = tryGetSavepointPathIfNeed(savepointRequest); - FlinkClusterClient clientWrapper = new FlinkClusterClient<>(client); - return clientWrapper - .triggerSavepoint(jobID, savepointPath, savepointRequest.nativeFormat()) - .get(); - } - - public void closeSubmit(SubmitRequest submitRequest, AutoCloseable... close) { - for (AutoCloseable resource : close) { - if (resource instanceof PackagedProgram) { - if (submitRequest.safePackageProgram()) { - Utils.close(resource); - } - } else { - Utils.close(resource); - } - } - } - - static Configuration safeSet(Configuration flinkConfig, ConfigOption option, T value) { - if (value != null && !value.toString().isEmpty()) { - flinkConfig.set(option, value); - } - return flinkConfig; - } - - private String tryGetSavepointPathIfNeed(SavepointRequestTrait request) { - if (!request.withSavepoint()) { - return null; - } - if (StringUtils.isNotBlank(request.savepointPath())) { - return request.savepointPath(); - } - String configDir = - getOptionFromDefaultFlinkConfig( - request.flinkVersion().getFlinkHome(), - org.apache.flink.configuration.ConfigOptions.key( - CheckpointingOptions.SAVEPOINT_DIRECTORY.key()) - .stringType() - .defaultValue( - request.deployMode() == FlinkDeployMode.YARN_APPLICATION - ? Workspace.remote().APP_SAVEPOINTS() - : null)); - - AssertUtils.required( - StringUtils.isNotBlank(configDir), - "[StreamPark] deployMode: " - + request.deployMode().getName() - + ", savePoint path is null or invalid."); - return configDir; - } - - private CommandLineAndConfig getCommandLineAndFlinkConfig(SubmitRequest submitRequest) throws Exception { - Options commandLineOptions = - getCommandLineOptions(submitRequest.flinkVersion().getFlinkHome()); - Map optionMap = buildOptionMap(submitRequest, commandLineOptions); - List cliArgs = buildCliArgs(submitRequest, optionMap); - logger().info("cliArgs: " + String.join(" ", cliArgs)); - CommandLine commandLine = - FlinkRunOption.parse(commandLineOptions, cliArgs.toArray(new String[0]), true); - - CustomCommandLine activeCommandLine = - validateAndGetActiveCommandLine( - getCustomCommandLines(submitRequest.flinkVersion().getFlinkHome()), commandLine); - - Configuration configuration = - applyConfiguration( - submitRequest.flinkVersion().getFlinkHome(), activeCommandLine, commandLine); - - return new CommandLineAndConfig(commandLine, configuration); - } - - private Map buildOptionMap( - SubmitRequest submitRequest, - Options commandLineOptions) { - Map optionMap = new HashMap<>(); - Map appOption = submitRequest.appOption(); - for (Map.Entry opt : appOption.entrySet()) { - if (!commandLineOptions.hasOption(opt.getKey())) { - logWarn("param:" + opt.getKey() + " is error,skip it."); - continue; - } - String option = commandLineOptions.getOption(opt.getKey().trim()).getOpt(); - Object parsedValue = parseOptionValue(opt.getValue()); - if (Boolean.TRUE.equals(parsedValue)) { - optionMap.put("-" + option, true); - } else if (!(parsedValue instanceof Boolean)) { - optionMap.put("-" + option, parsedValue); - } - } - - if (submitRequest.savePoint() != null) { - optionMap.put( - "-" + FlinkRunOption.SAVEPOINT_PATH_OPTION.getOpt(), - submitRequest.savePoint()); - } - - optionMap.remove("-e"); - optionMap.remove("--executor"); - optionMap.remove("-t"); - optionMap.remove("--target"); - if (submitRequest.deployMode() != null) { - optionMap.put("-t", submitRequest.deployMode().getName()); - } - return optionMap; - } - - private List buildCliArgs(SubmitRequest submitRequest, Map optionMap) { - List array = new ArrayList<>(); - for (Map.Entry opt : optionMap.entrySet()) { - array.add(opt.getKey()); - if (opt.getValue() instanceof String) { - array.add((String) opt.getValue()); - } - } - - if (MapUtils.isNotEmpty(submitRequest.properties())) { - for (Map.Entry key : submitRequest.properties().entrySet()) { - if (!key.getKey().startsWith("env.")) { - logInfo( - "application dynamicProperties: " - + key.getKey() - + " :" - + key.getValue()); - array.add("-D" + key.getKey() + "=" + key.getValue()); - } - } - } - return array; - } - - private List extractProgramArgs(SubmitRequest submitRequest) { - List programArgs = new ArrayList<>(); - programArgs.addAll(FlinkConfigurationUtils.extractArguments(submitRequest.args())); - - if (submitRequest.applicationType() == ApplicationType.STREAMPARK_FLINK) { - programArgs.add(paramKeyFlinkConf); - programArgs.add(submitRequest.flinkYaml()); - programArgs.add(paramKeyAppName); - programArgs.add(DeflaterUtils.zipString(submitRequest.effectiveAppName())); - programArgs.add(paramKeyFlinkParallelism); - programArgs.add(getParallelism(submitRequest).toString()); - - if (submitRequest.jobType() == FlinkJobType.FLINK_SQL) { - programArgs.add(paramKeyFlinkSql); - programArgs.add(submitRequest.flinkSQL()); - if (submitRequest.appConf() != null) { - programArgs.add(paramKeyAppConf); - programArgs.add(submitRequest.appConf()); - } - } else if (shouldAddAppConf(submitRequest.appConf())) { - programArgs.add(paramKeyAppConf); - programArgs.add(submitRequest.appConf()); - } - } else if (submitRequest.jobType() == FlinkJobType.FLINK_SQL) { - programArgs.add(paramKeyFlinkConf); - programArgs.add(submitRequest.flinkYaml()); - programArgs.add(paramKeyAppName); - programArgs.add(DeflaterUtils.zipString(submitRequest.effectiveAppName())); - programArgs.add(paramKeyFlinkParallelism); - programArgs.add(getParallelism(submitRequest).toString()); - programArgs.add(paramKeyFlinkSql); - programArgs.add(submitRequest.flinkSQL()); - if (submitRequest.appConf() != null) { - programArgs.add(paramKeyAppConf); - programArgs.add(submitRequest.appConf()); - } - } - - if (MapUtils.isNotEmpty(submitRequest.properties()) - && submitRequest.properties().containsKey(ExecutionOptions.RUNTIME_MODE.key())) { - String runtimeMode = - submitRequest.properties().get(ExecutionOptions.RUNTIME_MODE.key()).toString(); - programArgs.add("--" + ExecutionOptions.RUNTIME_MODE.key()); - programArgs.add(runtimeMode); - } - - if (submitRequest.jobType() == FlinkJobType.PYFLINK - && submitRequest.deployMode() != FlinkDeployMode.YARN_APPLICATION) { - programArgs.add("-py"); - programArgs.add(submitRequest.userJarFile().getAbsolutePath()); - } - - return Lists.newArrayList(programArgs); - } - - private Configuration applyConfiguration( - String flinkHome, CustomCommandLine activeCustomCommandLine, - CommandLine commandLine) throws Exception { - Preconditions.checkNotNull( - activeCustomCommandLine, "activeCustomCommandLine must not be null."); - Configuration configuration = new Configuration(); - Configuration flinkDefaultConfiguration = getFlinkDefaultConfiguration(flinkHome); - Set keys = flinkDefaultConfiguration.keySet(); - for (String key : keys) { - String value = flinkDefaultConfiguration.getString(key, null); - if (value != null) { - configuration.setString(key, value); - } - } - configuration.addAll(activeCustomCommandLine.toConfiguration(commandLine)); - return configuration; - } - - private static Object parseOptionValue(String value) { - if ("true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value)) { - return Boolean.parseBoolean(value); - } - return value; - } - - private static boolean shouldAddAppConf(String appConf) { - if (appConf == null) { - return true; - } - try { - return !appConf.startsWith("json:"); - } catch (Exception e) { - return true; - } - } - - private static String formatProperties(Map properties) { - if (MapUtils.isEmpty(properties)) { - return ""; - } - return properties.entrySet().stream() - .map(e -> e.getKey() + "=" + e.getValue()) - .collect(Collectors.joining(" ")); - } - - @FunctionalInterface - protected interface SubmitFunc { - - SubmitResponse apply(SubmitRequest request, Configuration config, File jarFile) throws FlinkException; - } - - private static final class CommandLineAndConfig { - - final CommandLine commandLine; - final Configuration flinkConfig; - - CommandLineAndConfig(CommandLine commandLine, Configuration flinkConfig) { - this.commandLine = commandLine; - this.flinkConfig = flinkConfig; - } - } -} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestResolverTest.java similarity index 64% rename from streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestTest.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestResolverTest.java index ae59b4eb71..649a9bd4d6 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestTest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestResolverTest.java @@ -17,14 +17,16 @@ package org.apache.streampark.flink.client.test; -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.conf.FlinkVersion; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.option.ApplicationOptions; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; -import org.apache.streampark.flink.client.bean.SubmitApplicationSpec; -import org.apache.streampark.flink.client.bean.SubmitRequest; +import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.request.SubmitApplicationSpec; +import org.apache.streampark.flink.client.request.SubmitRequest; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; @@ -36,20 +38,22 @@ import static org.assertj.core.api.Assertions.assertThat; -class SubmitRequestTest { +class SubmitRequestResolverTest { private static final FlinkVersion FLINK_VERSION = new FlinkVersion("/tmp/flink-home"); @Test void appMainForFlinkSqlJob() { SubmitRequest request = createRequest(FlinkJobType.FLINK_SQL, null); - assertThat(request.appMain()).isEqualTo(Constants.STREAMPARK_FLINKSQL_CLIENT_CLASS); + assertThat(SubmitRequestResolver.applicationMain(request)) + .isEqualTo(Constants.STREAMPARK_FLINKSQL_CLIENT_CLASS); } @Test void appMainForPyFlinkJob() { SubmitRequest request = createRequest(FlinkJobType.PYFLINK, null); - assertThat(request.appMain()).isEqualTo(Constants.PYTHON_FLINK_DRIVER_CLASS_NAME); + assertThat(SubmitRequestResolver.applicationMain(request)) + .isEqualTo(Constants.PYTHON_FLINK_DRIVER_CLASS_NAME); } @Test @@ -58,20 +62,21 @@ void appMainForFlinkJarFromJsonConf() { String appConf = String.format( "json://{\"%s\":\"%s\"}", - ConfigKeys.KEY_FLINK_APPLICATION_MAIN_CLASS(), mainClass); + FlinkOptions.APPLICATION_MAIN_CLASS.key(), mainClass); SubmitRequest request = createRequest(FlinkJobType.FLINK_JAR, appConf); - assertThat(request.appMain()).isEqualTo(mainClass); + assertThat(SubmitRequestResolver.applicationMain(request)).isEqualTo(mainClass); } @Test void savepointRestoreSettingsWithoutSavepoint() { SubmitRequest request = createRequest(FlinkJobType.FLINK_JAR, null); - assertThat(request.savepointRestoreSettings()).isEqualTo(SavepointRestoreSettings.none()); + assertThat(SubmitRequestResolver.savepointRestoreSettings(request)) + .isEqualTo(SavepointRestoreSettings.none()); } @Test void appPropertiesFromJsonConf() { - String propertyKey = ConfigKeys.KEY_FLINK_PROPERTY_PREFIX() + "parallelism.default"; + String propertyKey = FlinkOptions.PROPERTY_PREFIX.value() + "parallelism.default"; String appConf = "json://{\"" + propertyKey + "\":\"8\"}"; SubmitApplicationSpec application = SubmitApplicationSpec.builder() @@ -88,11 +93,12 @@ void appPropertiesFromJsonConf() { null, null); - assertThat(request.appProperties()).containsEntry("parallelism.default", "8"); + assertThat(SubmitRequestResolver.applicationProperties(request)) + .containsEntry("parallelism.default", "8"); } @Test - void propertiesMapCopiesSerializableIntegerValues() { + void copySerializableIntegers() { Map properties = new HashMap<>(); properties.put("key", 42); SubmitApplicationSpec application = @@ -107,19 +113,19 @@ void propertiesMapCopiesSerializableIntegerValues() { null, null); - assertThat(request.getProp("key")).isEqualTo(42); + assertThat(request.properties()).containsEntry("key", 42); } @Test - void flinkSqlShouldReturnNullWhenExtraParameterMissing() { + void returnNullWithoutFlinkSql() { SubmitRequest request = createRequest(FlinkJobType.FLINK_SQL, null); - assertThat(request.flinkSQL()).isNull(); + assertThat(SubmitRequestResolver.flinkSql(request)).isNull(); } @Test void flinkSqlShouldReadFromExtraParameter() { Map extra = new HashMap<>(); - extra.put(ConfigKeys.KEY_FLINK_SQL(), "select 1"); + extra.put(ApplicationOptions.SQL.key(), "select 1"); SubmitRequest request = new SubmitRequest( FLINK_VERSION, @@ -129,20 +135,39 @@ void flinkSqlShouldReadFromExtraParameter() { null, null, extra); - assertThat(request.flinkSQL()).isEqualTo("select 1"); + assertThat(SubmitRequestResolver.flinkSql(request)).isEqualTo("select 1"); } @Test - void getExtraShouldBeNullSafe() { + void extraParametersShouldDefaultToEmptyMap() { SubmitRequest request = createRequest(FlinkJobType.FLINK_JAR, null); - assertThat(request.getExtra("missing")).isNull(); - assertThat(request.hasExtra("missing")).isFalse(); + assertThat(request.extraParameter()).isEmpty(); } @Test - void allowNonRestoredStateShouldDefaultToFalse() { + void defaultAllowNonRestoredState() { SubmitRequest request = createRequest(FlinkJobType.FLINK_JAR, null); - assertThat(request.allowNonRestoredState()).isFalse(); + assertThat(SubmitRequestResolver.allowNonRestoredState(request)).isFalse(); + } + + @Test + void preserveYamlFormat() { + SubmitApplicationSpec application = + SubmitApplicationSpec.builder() + .jobType(FlinkJobType.FLINK_JAR) + .standardYaml(true) + .build(); + SubmitRequest request = + new SubmitRequest( + FLINK_VERSION, + FlinkDeployMode.YARN_APPLICATION, + Collections.emptyMap(), + application, + null, + null, + null); + + assertThat(request.standardYaml()).isTrue(); } private static SubmitRequest createRequest(FlinkJobType jobType, String appConf) { diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/YarnPerJobTestCase.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/YarnPerJobTestCase.java index 51950d5253..9a5fd21084 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/YarnPerJobTestCase.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/YarnPerJobTestCase.java @@ -17,9 +17,9 @@ package org.apache.streampark.flink.client.test; +import org.apache.streampark.common.configuration.FlinkRunOption; import org.apache.streampark.common.util.StreamParkLoggerFactory; -import org.apache.streampark.flink.client.bean.SubmitResponse; -import org.apache.streampark.flink.core.conf.FlinkRunOption; +import org.apache.streampark.flink.client.response.SubmitResponse; import org.apache.streampark.shaded.org.slf4j.Logger; diff --git a/streampark-flink/streampark-flink-kubernetes/pom.xml b/streampark-flink/streampark-flink-kubernetes/pom.xml index 1cc45a167b..1238cad8bc 100644 --- a/streampark-flink/streampark-flink-kubernetes/pom.xml +++ b/streampark-flink/streampark-flink-kubernetes/pom.xml @@ -26,7 +26,7 @@ streampark-flink-kubernetes - StreamPark : Flink Kubernetes Integration + StreamPark : Flink Kubernetes 5.1 diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/DefaultFlinkK8sWatcher.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/DefaultFlinkKubernetesWatcher.java similarity index 93% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/DefaultFlinkK8sWatcher.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/DefaultFlinkKubernetesWatcher.java index 854c2cb970..371450660e 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/DefaultFlinkK8sWatcher.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/DefaultFlinkKubernetesWatcher.java @@ -28,7 +28,7 @@ import org.apache.streampark.flink.kubernetes.model.TrackId; import org.apache.streampark.flink.kubernetes.watcher.FlinkCheckpointWatcher; import org.apache.streampark.flink.kubernetes.watcher.FlinkJobStatusWatcher; -import org.apache.streampark.flink.kubernetes.watcher.FlinkK8sEventWatcher; +import org.apache.streampark.flink.kubernetes.watcher.FlinkKubernetesEventWatcher; import org.apache.streampark.flink.kubernetes.watcher.FlinkMetricWatcher; import org.apache.streampark.flink.kubernetes.watcher.FlinkWatcher; @@ -42,22 +42,22 @@ import java.util.Set; /** Default K8sFlinkTrackMonitor implementation. */ -public class DefaultFlinkK8sWatcher extends LoggerSupport implements FlinkK8sWatcher { +public class DefaultFlinkKubernetesWatcher extends LoggerSupport implements FlinkKubernetesWatcher { - public final FlinkK8sWatchController watchController; + public final FlinkKubernetesWatchController watchController; private final ChangeEventBus eventBus; - private final FlinkK8sEventWatcher k8sEventWatcher; + private final FlinkKubernetesEventWatcher k8sEventWatcher; private final FlinkJobStatusWatcher jobStatusWatcher; private final FlinkMetricWatcher metricsWatcher; private final FlinkCheckpointWatcher checkpointWatcher; private final FlinkWatcher[] allWatchers; - public DefaultFlinkK8sWatcher(FlinkTrackConfig conf) { - this.watchController = new FlinkK8sWatchController(conf.jobStatusWatcherConf()); + public DefaultFlinkKubernetesWatcher(FlinkTrackConfig conf) { + this.watchController = new FlinkKubernetesWatchController(conf.jobStatusWatcherConf()); this.eventBus = new ChangeEventBus(); this.eventBus.registerListener(new BuildInEventListener()); - this.k8sEventWatcher = new FlinkK8sEventWatcher(watchController); + this.k8sEventWatcher = new FlinkKubernetesEventWatcher(watchController); this.jobStatusWatcher = new FlinkJobStatusWatcher( conf.jobStatusWatcherConf(), watchController, eventBus); diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkK8sWatchController.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkKubernetesWatchController.java similarity index 93% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkK8sWatchController.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkKubernetesWatchController.java index bbfad205ec..2a725f0473 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkK8sWatchController.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkKubernetesWatchController.java @@ -29,16 +29,16 @@ import java.util.stream.Collectors; /** Tracking info cache pool on flink kubernetes mode. */ -public class FlinkK8sWatchController extends LoggerSupport implements AutoCloseable { +public class FlinkKubernetesWatchController extends LoggerSupport implements AutoCloseable { public final TrackIdCache trackIds = TrackIdCache.build(); public final TrackIdCache canceling = TrackIdCache.build(); public final EndpointCache endpoints = EndpointCache.build(); public final JobStatusCache jobStatuses; - public final K8sDeploymentEventCache k8sDeploymentEvents = K8sDeploymentEventCache.build(); + public final KubernetesDeploymentEventCache k8sDeploymentEvents = KubernetesDeploymentEventCache.build(); public final MetricCache flinkMetrics = MetricCache.build(); - public FlinkK8sWatchController(JobStatusWatcherConfig conf) { + public FlinkKubernetesWatchController(JobStatusWatcherConfig conf) { this.jobStatuses = JobStatusCache.build(conf.jobStatusCacheTimeOutSec()); } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkK8sWatcher.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkKubernetesWatcher.java similarity index 98% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkK8sWatcher.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkKubernetesWatcher.java index 63c7202839..468e922dbc 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkK8sWatcher.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkKubernetesWatcher.java @@ -37,7 +37,7 @@ * streampark.flink.kubernetes package. */ @Public -public interface FlinkK8sWatcher extends AutoCloseable { +public interface FlinkKubernetesWatcher extends AutoCloseable { /** * Register listener to EventBus. diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkK8sWatcherFactory.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkKubernetesWatcherFactory.java similarity index 74% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkK8sWatcherFactory.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkKubernetesWatcherFactory.java index e31916941f..985b23c214 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkK8sWatcherFactory.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkKubernetesWatcherFactory.java @@ -20,9 +20,9 @@ import org.apache.flink.annotation.Public; @Public -public final class FlinkK8sWatcherFactory { +public final class FlinkKubernetesWatcherFactory { - private FlinkK8sWatcherFactory() { + private FlinkKubernetesWatcherFactory() { } /** @@ -33,19 +33,19 @@ private FlinkK8sWatcherFactory() { * case, there is no need to display the call to FlinkK8sWatcher.start(), useless the monitor * is expected to start immediately. */ - public static FlinkK8sWatcher createInstance(FlinkTrackConfig conf, boolean lazyStart) { - DefaultFlinkK8sWatcher watcher = new DefaultFlinkK8sWatcher(conf); + public static FlinkKubernetesWatcher createInstance(FlinkTrackConfig conf, boolean lazyStart) { + DefaultFlinkKubernetesWatcher watcher = new DefaultFlinkKubernetesWatcher(conf); if (lazyStart) { - return new LazyStartFlinkK8sWatcher(watcher); + return new LazyStartFlinkKubernetesWatcher(watcher); } return watcher; } - public static FlinkK8sWatcher createInstance() { + public static FlinkKubernetesWatcher createInstance() { return createInstance(FlinkTrackConfig.defaultConf(), false); } - public static FlinkK8sWatcher createInstance(FlinkTrackConfig conf) { + public static FlinkKubernetesWatcher createInstance(FlinkTrackConfig conf) { return createInstance(conf, false); } } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkTrackConfig.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkTrackConfig.java index 0f5826e8ad..2035cb51a8 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkTrackConfig.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/FlinkTrackConfig.java @@ -17,8 +17,9 @@ package org.apache.streampark.flink.kubernetes; -import org.apache.streampark.common.conf.InternalConfigHolder; -import org.apache.streampark.common.conf.K8sFlinkConfig; +import org.apache.streampark.common.configuration.Configuration; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.flink.kubernetes.configuration.KubernetesOptions; import lombok.AllArgsConstructor; import lombok.Builder; @@ -56,21 +57,33 @@ public static FlinkTrackConfig debugConf() { .build(); } - /** create from ConfigHub */ - public static FlinkTrackConfig fromConfigHub() { + /** Creates tracking configuration from the current immutable server snapshot. */ + public static FlinkTrackConfig fromConfiguration() { + Configuration configuration = GlobalConfiguration.current(); return FlinkTrackConfig.builder() .jobStatusWatcherConf( JobStatusWatcherConfig.builder() - .requestTimeoutSec(InternalConfigHolder.get(K8sFlinkConfig.jobStatusTrackTaskTimeoutSec)) - .requestIntervalSec(InternalConfigHolder.get(K8sFlinkConfig.jobStatueTrackTaskIntervalSec)) + .requestTimeoutSec( + configuration.get(KubernetesOptions.JOB_STATUS_REQUEST_TIMEOUT).getSeconds()) + .requestIntervalSec( + configuration.get(KubernetesOptions.JOB_STATUS_POLL_INTERVAL).getSeconds()) .silentStateJobKeepTrackingSec( - InternalConfigHolder.get(K8sFlinkConfig.silentStateJobKeepTrackingSec)) - .jobStatusCacheTimeOutSec(InternalConfigHolder.get(K8sFlinkConfig.jobStatusTrackCacheTimeoutSec)) + Math.toIntExact( + configuration + .get(KubernetesOptions.SILENT_STATE_TRACKING_RETENTION) + .getSeconds())) + .jobStatusCacheTimeOutSec( + Math.toIntExact( + configuration + .get(KubernetesOptions.JOB_STATUS_CACHE_TIMEOUT) + .getSeconds())) .build()) .metricWatcherConf( MetricWatcherConfig.builder() - .requestTimeoutSec(InternalConfigHolder.get(K8sFlinkConfig.metricTrackTaskTimeoutSec)) - .requestIntervalSec(InternalConfigHolder.get(K8sFlinkConfig.metricTrackTaskIntervalSec)) + .requestTimeoutSec( + configuration.get(KubernetesOptions.METRIC_REQUEST_TIMEOUT).getSeconds()) + .requestIntervalSec( + configuration.get(KubernetesOptions.METRIC_POLL_INTERVAL).getSeconds()) .build()) .build(); } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/K8sDeploymentEventCache.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/KubernetesDeploymentEventCache.java similarity index 65% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/K8sDeploymentEventCache.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/KubernetesDeploymentEventCache.java index 8f8ae71b64..057bc322c6 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/K8sDeploymentEventCache.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/KubernetesDeploymentEventCache.java @@ -17,8 +17,8 @@ package org.apache.streampark.flink.kubernetes; -import org.apache.streampark.flink.kubernetes.model.K8sDeploymentEventCV; -import org.apache.streampark.flink.kubernetes.model.K8sEventKey; +import org.apache.streampark.flink.kubernetes.model.KubernetesDeploymentEventCV; +import org.apache.streampark.flink.kubernetes.model.KubernetesEventKey; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -26,19 +26,19 @@ import java.util.HashMap; import java.util.Map; -public class K8sDeploymentEventCache { +public class KubernetesDeploymentEventCache { - final Cache cache = Caffeine.newBuilder().build(); + final Cache cache = Caffeine.newBuilder().build(); - public void put(K8sEventKey key, K8sDeploymentEventCV value) { + public void put(KubernetesEventKey key, KubernetesDeploymentEventCV value) { cache.put(key, value); } - public K8sDeploymentEventCV get(K8sEventKey key) { + public KubernetesDeploymentEventCV get(KubernetesEventKey key) { return cache.getIfPresent(key); } - public Map asMap() { + public Map asMap() { return new HashMap<>(cache.asMap()); } @@ -46,7 +46,7 @@ public void cleanUp() { cache.cleanUp(); } - public static K8sDeploymentEventCache build() { - return new K8sDeploymentEventCache(); + public static KubernetesDeploymentEventCache build() { + return new KubernetesDeploymentEventCache(); } } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/K8sPodTemplateFiles.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/KubernetesPodTemplateFiles.java similarity index 93% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/K8sPodTemplateFiles.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/KubernetesPodTemplateFiles.java index ed7fe0ecdb..4d208d22f1 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/K8sPodTemplateFiles.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/KubernetesPodTemplateFiles.java @@ -24,11 +24,11 @@ /** * @param tmplFiles key of flink pod template configuration -> absolute file path of pod template */ -public class K8sPodTemplateFiles { +public class KubernetesPodTemplateFiles { private final Map tmplFiles; - public K8sPodTemplateFiles(Map tmplFiles) { + public KubernetesPodTemplateFiles(Map tmplFiles) { this.tmplFiles = tmplFiles; } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/KubernetesRetriever.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/KubernetesRetriever.java index 239594eb42..ff27fd1210 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/KubernetesRetriever.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/KubernetesRetriever.java @@ -19,7 +19,7 @@ import org.apache.streampark.common.util.AutoCloseUtils; import org.apache.streampark.common.util.LoggerSupport; -import org.apache.streampark.flink.kubernetes.enums.FlinkK8sDeployMode; +import org.apache.streampark.flink.kubernetes.enums.FlinkKubernetesDeployMode; import org.apache.streampark.flink.kubernetes.ingress.IngressController; import org.apache.streampark.flink.kubernetes.model.ClusterKey; @@ -79,7 +79,7 @@ public static boolean checkK8sConnection() { public static Optional> newFinkClusterClient( String clusterId, @Nullable String namespace, - FlinkK8sDeployMode executeMode) { + FlinkKubernetesDeployMode executeMode) { Configuration flinkConfig = new Configuration(); flinkConfig.setString(DeploymentOptions.TARGET, executeMode.toString()); flinkConfig.setString(KubernetesConfigOptions.CLUSTER_ID, clusterId); diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/LazyStartFlinkK8sWatcher.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/LazyStartFlinkKubernetesWatcher.java similarity index 94% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/LazyStartFlinkK8sWatcher.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/LazyStartFlinkKubernetesWatcher.java index b8a2ee0bf7..a6c522d8e1 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/LazyStartFlinkK8sWatcher.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/LazyStartFlinkKubernetesWatcher.java @@ -33,11 +33,11 @@ * Decorator for FlinkK8sWatcher used to trigger the run behavior. What more, this decorator has * the ability to automatically recover the FlinkK8sWatcher's internal FlinkWatcher. */ -public class LazyStartFlinkK8sWatcher implements FlinkK8sWatcher { +public class LazyStartFlinkKubernetesWatcher implements FlinkKubernetesWatcher { - private final DefaultFlinkK8sWatcher delegate; + private final DefaultFlinkKubernetesWatcher delegate; - public LazyStartFlinkK8sWatcher(DefaultFlinkK8sWatcher delegate) { + public LazyStartFlinkKubernetesWatcher(DefaultFlinkKubernetesWatcher delegate) { this.delegate = delegate; } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/PodTemplateTool.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/PodTemplateTool.java index 66663f3dd5..6f6bab8f97 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/PodTemplateTool.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/PodTemplateTool.java @@ -17,8 +17,8 @@ package org.apache.streampark.flink.kubernetes; -import org.apache.streampark.flink.kubernetes.model.K8sPodTemplates; -import org.apache.streampark.spark.kubernetes.model.SparkK8sPodTemplates; +import org.apache.streampark.flink.kubernetes.model.KubernetesPodTemplates; +import org.apache.streampark.spark.kubernetes.model.SparkKubernetesPodTemplates; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; @@ -56,9 +56,9 @@ private PodTemplateTool() { * @param podTemplates flink kubernetes pod templates * @return Map[k8s pod template option, template file output path] */ - public static K8sPodTemplateFiles preparePodTemplateFiles( - String buildWorkspace, - K8sPodTemplates podTemplates) throws IOException { + public static KubernetesPodTemplateFiles preparePodTemplateFiles( + String buildWorkspace, + KubernetesPodTemplates podTemplates) throws IOException { File workspaceDir = new File(buildWorkspace); if (!workspaceDir.exists()) { workspaceDir.mkdir(); @@ -68,7 +68,7 @@ public static K8sPodTemplateFiles preparePodTemplateFiles( writeTemplate(buildWorkspace, podTemplates.podTemplate(), KUBERNETES_POD_TEMPLATE, podTempleMap); writeTemplate(buildWorkspace, podTemplates.jmPodTemplate(), KUBERNETES_JM_POD_TEMPLATE, podTempleMap); writeTemplate(buildWorkspace, podTemplates.tmPodTemplate(), KUBERNETES_TM_POD_TEMPLATE, podTempleMap); - return new K8sPodTemplateFiles(Collections.unmodifiableMap(podTempleMap)); + return new KubernetesPodTemplateFiles(Collections.unmodifiableMap(podTempleMap)); } /** @@ -78,9 +78,9 @@ public static K8sPodTemplateFiles preparePodTemplateFiles( * @param podTemplates spark kubernetes pod templates * @return Map[k8s pod template option, template file output path] */ - public static K8sPodTemplateFiles preparePodTemplateFiles( - String buildWorkspace, - SparkK8sPodTemplates podTemplates) throws IOException { + public static KubernetesPodTemplateFiles preparePodTemplateFiles( + String buildWorkspace, + SparkKubernetesPodTemplates podTemplates) throws IOException { File workspaceDir = new File(buildWorkspace); if (!workspaceDir.exists()) { workspaceDir.mkdir(); @@ -90,7 +90,7 @@ public static K8sPodTemplateFiles preparePodTemplateFiles( writeTemplate(buildWorkspace, podTemplates.driverPodTemplate(), KUBERNETES_DRIVER_POD_TEMPLATE, podTempleMap); writeTemplate( buildWorkspace, podTemplates.executorPodTemplate(), KUBERNETES_EXECUTOR_POD_TEMPLATE, podTempleMap); - return new K8sPodTemplateFiles(Collections.unmodifiableMap(podTempleMap)); + return new KubernetesPodTemplateFiles(Collections.unmodifiableMap(podTempleMap)); } private static void writeTemplate( diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/configuration/KubernetesOptions.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/configuration/KubernetesOptions.java new file mode 100644 index 0000000000..711e1a7f73 --- /dev/null +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/configuration/KubernetesOptions.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.kubernetes.configuration; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; + +import java.time.Duration; +import java.time.temporal.ChronoUnit; + +/** Options owned by StreamPark's Flink Kubernetes integration. */ +public final class KubernetesOptions { + + public static final ConfigOption JOB_STATUS_REQUEST_TIMEOUT = + durationOption( + "streampark.flink-k8s.tracking.polling-task-timeout-sec.job-status", + Duration.ofSeconds(120), + "Timeout for one job-status polling request."); + + public static final ConfigOption JOB_STATUS_CACHE_TIMEOUT = + durationOption( + "streampark.flink-k8s.tracking.cache-timeout-sec.job-status", + Duration.ofSeconds(300), + "Lifetime of one cached job-status entry."); + + public static final ConfigOption METRIC_REQUEST_TIMEOUT = + durationOption( + "streampark.flink-k8s.tracking.polling-task-timeout-sec.cluster-metric", + Duration.ofSeconds(120), + "Timeout for one cluster-metric polling request."); + + public static final ConfigOption JOB_STATUS_POLL_INTERVAL = + durationOption( + "streampark.flink-k8s.tracking.polling-interval-sec.job-status", + Duration.ofSeconds(5), + "Interval between job-status polling requests."); + + public static final ConfigOption METRIC_POLL_INTERVAL = + durationOption( + "streampark.flink-k8s.tracking.polling-interval-sec.cluster-metric", + Duration.ofSeconds(5), + "Interval between cluster-metric polling requests."); + + public static final ConfigOption SILENT_STATE_TRACKING_RETENTION = + durationOption( + "streampark.flink-k8s.tracking.silent-state-keep-sec", + Duration.ofSeconds(60), + "How long a job in SILENT state remains tracked."); + + public static final ConfigOption INGRESS_CLASS = + ConfigOptions.key("streampark.flink-k8s.ingress.class") + .stringType() + .defaultValue("nginx") + .check(value -> !value.trim().isEmpty(), "ingress class must not be blank") + .withDescription("Ingress controller class used by generated Kubernetes resources.") + .build(); + + private KubernetesOptions() { + } + + private static ConfigOption durationOption( + String key, + Duration defaultValue, + String description) { + return ConfigOptions.key(key) + .durationType(ChronoUnit.SECONDS) + .defaultValue(defaultValue) + .check(value -> value.getSeconds() > 0, "duration must be at least one second") + .withDescription(description) + .build(); + } +} diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/configuration/package-info.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/configuration/package-info.java new file mode 100644 index 0000000000..a1fdf22edb --- /dev/null +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/configuration/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Typed options owned by StreamPark's Flink Kubernetes integration. */ +package org.apache.streampark.flink.kubernetes.configuration; diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/enums/FlinkK8sDeployMode.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/enums/FlinkKubernetesDeployMode.java similarity index 85% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/enums/FlinkK8sDeployMode.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/enums/FlinkKubernetesDeployMode.java index f76acf8b7e..3d4e697b44 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/enums/FlinkK8sDeployMode.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/enums/FlinkKubernetesDeployMode.java @@ -23,20 +23,20 @@ import com.fasterxml.jackson.annotation.JsonValue; /** execution mode of flink on kubernetes */ -public enum FlinkK8sDeployMode { +public enum FlinkKubernetesDeployMode { SESSION("kubernetes-session"), APPLICATION("kubernetes-application"); private final String value; - FlinkK8sDeployMode(String value) { + FlinkKubernetesDeployMode(String value) { this.value = value; } @JsonCreator - public static FlinkK8sDeployMode of(String value) { - for (FlinkK8sDeployMode mode : FlinkK8sDeployMode.values()) { + public static FlinkKubernetesDeployMode of(String value) { + for (FlinkKubernetesDeployMode mode : FlinkKubernetesDeployMode.values()) { if (mode.value.equals(value)) { return mode; } @@ -44,7 +44,7 @@ public static FlinkK8sDeployMode of(String value) { throw new IllegalArgumentException("Illegal FlinkK8sDeployMode value: " + value); } - public static FlinkK8sDeployMode of(FlinkDeployMode mode) { + public static FlinkKubernetesDeployMode of(FlinkDeployMode mode) { switch (mode) { case KUBERNETES_NATIVE_SESSION: return SESSION; @@ -55,7 +55,7 @@ public static FlinkK8sDeployMode of(FlinkDeployMode mode) { } } - public static FlinkDeployMode toFlinkDeployMode(FlinkK8sDeployMode mode) { + public static FlinkDeployMode toFlinkDeployMode(FlinkKubernetesDeployMode mode) { switch (mode) { case SESSION: return FlinkDeployMode.KUBERNETES_NATIVE_SESSION; diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/helper/KubernetesWatcherHelper.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/helper/KubernetesWatcherHelper.java index 8a1a591e4c..4c56e0caeb 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/helper/KubernetesWatcherHelper.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/helper/KubernetesWatcherHelper.java @@ -18,8 +18,8 @@ package org.apache.streampark.flink.kubernetes.helper; import org.apache.streampark.common.util.LoggerSupport; -import org.apache.streampark.flink.kubernetes.DefaultFlinkK8sWatcher; -import org.apache.streampark.flink.kubernetes.FlinkK8sWatcher; +import org.apache.streampark.flink.kubernetes.DefaultFlinkKubernetesWatcher; +import org.apache.streampark.flink.kubernetes.FlinkKubernetesWatcher; import java.util.Timer; import java.util.TimerTask; @@ -35,7 +35,7 @@ private KubernetesWatcherHelper() { } // print job status cache size info - public static void watchJobStatusCacheSize(FlinkK8sWatcher k8sWatcher) { + public static void watchJobStatusCacheSize(FlinkKubernetesWatcher k8sWatcher) { new Timer() .scheduleAtFixedRate( timerTask( @@ -49,7 +49,7 @@ public static void watchJobStatusCacheSize(FlinkK8sWatcher k8sWatcher) { } // print agg flink cluster metrics cache detail - public static void watchAggClusterMetricsCache(FlinkK8sWatcher k8sWatcher) { + public static void watchAggClusterMetricsCache(FlinkKubernetesWatcher k8sWatcher) { new Timer() .scheduleAtFixedRate( timerTask( @@ -63,8 +63,8 @@ public static void watchAggClusterMetricsCache(FlinkK8sWatcher k8sWatcher) { } // print all cluster metrics for each flink cluster - public static void watchClusterMetricsCache(FlinkK8sWatcher k8sWatcher) { - DefaultFlinkK8sWatcher watcher = (DefaultFlinkK8sWatcher) k8sWatcher; + public static void watchClusterMetricsCache(FlinkKubernetesWatcher k8sWatcher) { + DefaultFlinkKubernetesWatcher watcher = (DefaultFlinkKubernetesWatcher) k8sWatcher; new Timer() .scheduleAtFixedRate( timerTask( @@ -80,7 +80,7 @@ public static void watchClusterMetricsCache(FlinkK8sWatcher k8sWatcher) { } // print job cache detail - public static void watchJobStatusCache(FlinkK8sWatcher k8sWatcher) { + public static void watchJobStatusCache(FlinkKubernetesWatcher k8sWatcher) { new Timer() .scheduleAtFixedRate( timerTask( @@ -96,7 +96,7 @@ public static void watchJobStatusCache(FlinkK8sWatcher k8sWatcher) { } // print trackId cache detail - public static void watchTrackIdsCache(FlinkK8sWatcher k8sWatcher) { + public static void watchTrackIdsCache(FlinkKubernetesWatcher k8sWatcher) { new Timer() .scheduleAtFixedRate( timerTask( @@ -109,7 +109,7 @@ public static void watchTrackIdsCache(FlinkK8sWatcher k8sWatcher) { 1500); } - public static void watchTrackIdsCacheSize(FlinkK8sWatcher k8sWatcher) { + public static void watchTrackIdsCacheSize(FlinkKubernetesWatcher k8sWatcher) { new Timer() .scheduleAtFixedRate( timerTask( @@ -122,8 +122,8 @@ public static void watchTrackIdsCacheSize(FlinkK8sWatcher k8sWatcher) { 1500); } - public static void watchK8sEventCache(FlinkK8sWatcher k8sWatcher) { - DefaultFlinkK8sWatcher watcher = (DefaultFlinkK8sWatcher) k8sWatcher; + public static void watchK8sEventCache(FlinkKubernetesWatcher k8sWatcher) { + DefaultFlinkKubernetesWatcher watcher = (DefaultFlinkKubernetesWatcher) k8sWatcher; new Timer() .scheduleAtFixedRate( timerTask( diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategy.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategy.java index 4e626242f7..142e68174e 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategy.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/ingress/IngressStrategy.java @@ -17,10 +17,9 @@ package org.apache.streampark.flink.kubernetes.ingress; -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.conf.InternalConfigHolder; -import org.apache.streampark.common.conf.K8sFlinkConfig; +import org.apache.streampark.common.configuration.GlobalConfiguration; import org.apache.streampark.common.util.FileUtils; +import org.apache.streampark.flink.kubernetes.configuration.KubernetesOptions; import org.apache.flink.client.program.ClusterClient; import org.apache.flink.kubernetes.shaded.io.fabric8.kubernetes.api.model.OwnerReference; @@ -35,9 +34,10 @@ public interface IngressStrategy { String REST_SERVICE_IDENTIFICATION = "rest"; + String FLINK_NATIVE_KUBERNETES_LABEL = "flink-native-kubernetes"; default String ingressClass() { - return InternalConfigHolder.get(K8sFlinkConfig.ingressClass); + return GlobalConfiguration.current().get(KubernetesOptions.INGRESS_CLASS); } String getIngressUrl(String nameSpace, String clusterId, ClusterClient clusterClient); @@ -77,7 +77,7 @@ default Map buildIngressAnnotations(String clusterId, String nam default Map buildIngressLabels(String clusterId) { Map labels = new HashMap<>(); labels.put("app", clusterId); - labels.put("type", ConfigKeys.FLINK_NATIVE_KUBERNETES_LABEL()); + labels.put("type", FLINK_NATIVE_KUBERNETES_LABEL); labels.put("component", "ingress"); return labels; } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/ClusterKey.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/ClusterKey.java index a50fc0cb31..1390fd97f7 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/ClusterKey.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/ClusterKey.java @@ -18,7 +18,7 @@ package org.apache.streampark.flink.kubernetes.model; import org.apache.streampark.common.util.Utils; -import org.apache.streampark.flink.kubernetes.enums.FlinkK8sDeployMode; +import org.apache.streampark.flink.kubernetes.enums.FlinkKubernetesDeployMode; import lombok.AllArgsConstructor; import lombok.Builder; @@ -28,12 +28,12 @@ @AllArgsConstructor public class ClusterKey { - private final FlinkK8sDeployMode executeMode; + private final FlinkKubernetesDeployMode executeMode; @Builder.Default private final String namespace = "default"; private final String clusterId; - public FlinkK8sDeployMode executeMode() { + public FlinkKubernetesDeployMode executeMode() { return executeMode; } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/K8sDeploymentEventCV.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/KubernetesDeploymentEventCV.java similarity index 97% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/K8sDeploymentEventCV.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/KubernetesDeploymentEventCV.java index a50eb8ca6f..02d75fcd89 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/K8sDeploymentEventCV.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/KubernetesDeploymentEventCV.java @@ -30,7 +30,7 @@ */ @Builder @AllArgsConstructor -public class K8sDeploymentEventCV { +public class KubernetesDeploymentEventCV { private final Action action; private final Deployment event; diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/K8sEventKey.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/KubernetesEventKey.java similarity index 91% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/K8sEventKey.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/KubernetesEventKey.java index b931ffeb45..2525609ded 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/K8sEventKey.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/KubernetesEventKey.java @@ -24,7 +24,7 @@ @Builder @AllArgsConstructor -public class K8sEventKey { +public class KubernetesEventKey { private final String namespace; private final String clusterId; @@ -44,10 +44,10 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof K8sEventKey)) { + if (!(obj instanceof KubernetesEventKey)) { return false; } - K8sEventKey that = (K8sEventKey) obj; + KubernetesEventKey that = (KubernetesEventKey) obj; return namespace.equals(that.namespace) && clusterId.equals(that.clusterId); } } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/K8sPodTemplates.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/KubernetesPodTemplates.java similarity index 86% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/K8sPodTemplates.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/KubernetesPodTemplates.java index 10d0622e73..fcc85d0e19 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/K8sPodTemplates.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/KubernetesPodTemplates.java @@ -25,7 +25,7 @@ /** Pod template for flink k8s cluster */ @Builder @AllArgsConstructor -public class K8sPodTemplates { +public class KubernetesPodTemplates { @Builder.Default private final String podTemplate = ""; @@ -54,12 +54,12 @@ public boolean isEmpty() { return !nonEmpty(); } - public static K8sPodTemplates empty() { - return K8sPodTemplates.builder().build(); + public static KubernetesPodTemplates empty() { + return KubernetesPodTemplates.builder().build(); } - public static K8sPodTemplates of(String podTemplate, String jmPodTemplate, String tmPodTemplate) { - return K8sPodTemplates.builder() + public static KubernetesPodTemplates of(String podTemplate, String jmPodTemplate, String tmPodTemplate) { + return KubernetesPodTemplates.builder() .podTemplate(safeGet(podTemplate)) .jmPodTemplate(safeGet(jmPodTemplate)) .tmPodTemplate(safeGet(tmPodTemplate)) @@ -91,10 +91,10 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof K8sPodTemplates)) { + if (!(obj instanceof KubernetesPodTemplates)) { return false; } - K8sPodTemplates that = (K8sPodTemplates) obj; + KubernetesPodTemplates that = (KubernetesPodTemplates) obj; return trimSafe(podTemplate).equals(trimSafe(that.podTemplate)) && trimSafe(jmPodTemplate).equals(trimSafe(that.jmPodTemplate)) && trimSafe(tmPodTemplate).equals(trimSafe(that.tmPodTemplate)); diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/TrackId.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/TrackId.java index b4ab58980c..a377602caa 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/TrackId.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/TrackId.java @@ -18,7 +18,7 @@ package org.apache.streampark.flink.kubernetes.model; import org.apache.streampark.common.util.Utils; -import org.apache.streampark.flink.kubernetes.enums.FlinkK8sDeployMode; +import org.apache.streampark.flink.kubernetes.enums.FlinkKubernetesDeployMode; import lombok.AllArgsConstructor; import lombok.Builder; @@ -30,7 +30,7 @@ @AllArgsConstructor public class TrackId { - private final FlinkK8sDeployMode executeMode; + private final FlinkKubernetesDeployMode executeMode; @Builder.Default private final String namespace = "default"; private final String clusterId; @@ -39,7 +39,7 @@ public class TrackId { private final String groupId; private final Properties properties; - public FlinkK8sDeployMode executeMode() { + public FlinkKubernetesDeployMode executeMode() { return executeMode; } @@ -99,7 +99,7 @@ public static TrackId onSession( String groupId, Properties properties) { return TrackId.builder() - .executeMode(FlinkK8sDeployMode.SESSION) + .executeMode(FlinkKubernetesDeployMode.SESSION) .namespace(namespace) .clusterId(clusterId) .appId(appId) @@ -117,7 +117,7 @@ public static TrackId onApplication( String groupId, Properties properties) { return TrackId.builder() - .executeMode(FlinkK8sDeployMode.APPLICATION) + .executeMode(FlinkKubernetesDeployMode.APPLICATION) .namespace(namespace) .clusterId(clusterId) .appId(appId) diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkCheckpointWatcher.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkCheckpointWatcher.java index bb33cee2c0..7876a46ffa 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkCheckpointWatcher.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkCheckpointWatcher.java @@ -18,7 +18,7 @@ package org.apache.streampark.flink.kubernetes.watcher; import org.apache.streampark.flink.kubernetes.ChangeEventBus; -import org.apache.streampark.flink.kubernetes.FlinkK8sWatchController; +import org.apache.streampark.flink.kubernetes.FlinkKubernetesWatchController; import org.apache.streampark.flink.kubernetes.KubernetesRetriever; import org.apache.streampark.flink.kubernetes.MetricWatcherConfig; import org.apache.streampark.flink.kubernetes.event.FlinkJobCheckpointChangeEvent; @@ -46,14 +46,14 @@ public class FlinkCheckpointWatcher extends FlinkWatcher { private final MetricWatcherConfig conf; - private final FlinkK8sWatchController watchController; + private final FlinkKubernetesWatchController watchController; private final ChangeEventBus eventBus; private ScheduledFuture timerSchedule; public FlinkCheckpointWatcher( MetricWatcherConfig conf, - FlinkK8sWatchController watchController, + FlinkKubernetesWatchController watchController, ChangeEventBus eventBus) { this.conf = conf; this.watchController = watchController; diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkJobStatusWatcher.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkJobStatusWatcher.java index 6465653072..727b43ed77 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkJobStatusWatcher.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkJobStatusWatcher.java @@ -18,11 +18,11 @@ package org.apache.streampark.flink.kubernetes.watcher; import org.apache.streampark.flink.kubernetes.ChangeEventBus; -import org.apache.streampark.flink.kubernetes.FlinkK8sWatchController; +import org.apache.streampark.flink.kubernetes.FlinkKubernetesWatchController; import org.apache.streampark.flink.kubernetes.JobStatusWatcherConfig; import org.apache.streampark.flink.kubernetes.KubernetesRetriever; import org.apache.streampark.flink.kubernetes.enums.FlinkJobState; -import org.apache.streampark.flink.kubernetes.enums.FlinkK8sDeployMode; +import org.apache.streampark.flink.kubernetes.enums.FlinkKubernetesDeployMode; import org.apache.streampark.flink.kubernetes.event.FlinkJobStatusChangeEvent; import org.apache.streampark.flink.kubernetes.helper.KubernetesDeploymentHelper; import org.apache.streampark.flink.kubernetes.model.ClusterKey; @@ -62,14 +62,14 @@ public class FlinkJobStatusWatcher extends FlinkWatcher { private static final String TRACK_IDS_SUFFIX = ", trackIds="; private final JobStatusWatcherConfig conf; - private final FlinkK8sWatchController watchController; + private final FlinkKubernetesWatchController watchController; private final ChangeEventBus eventBus; private ScheduledFuture timerSchedule; public FlinkJobStatusWatcher( JobStatusWatcherConfig conf, - FlinkK8sWatchController watchController, + FlinkKubernetesWatchController watchController, ChangeEventBus eventBus) { this.conf = conf; this.watchController = watchController; @@ -112,7 +112,7 @@ public void doWatch() { Set>> appFuture = trackIds.stream() - .filter(id -> id.executeMode() == FlinkK8sDeployMode.APPLICATION) + .filter(id -> id.executeMode() == FlinkKubernetesDeployMode.APPLICATION) .map( id -> CompletableFuture .supplyAsync(() -> touchApplicationJob(id), watchExecutor) @@ -129,7 +129,7 @@ public void doWatch() { Set sessionIds = trackIds.stream() - .filter(id -> id.executeMode() == FlinkK8sDeployMode.SESSION) + .filter(id -> id.executeMode() == FlinkKubernetesDeployMode.SESSION) .collect(Collectors.toSet()); Set sessionCluster = new HashSet<>(); for (TrackId sessionId : sessionIds) { diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkK8sEventWatcher.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkKubernetesEventWatcher.java similarity index 85% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkK8sEventWatcher.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkKubernetesEventWatcher.java index e35e4f7196..9f6b2fe018 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkK8sEventWatcher.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkKubernetesEventWatcher.java @@ -17,10 +17,10 @@ package org.apache.streampark.flink.kubernetes.watcher; -import org.apache.streampark.flink.kubernetes.FlinkK8sWatchController; +import org.apache.streampark.flink.kubernetes.FlinkKubernetesWatchController; import org.apache.streampark.flink.kubernetes.KubernetesRetriever; -import org.apache.streampark.flink.kubernetes.model.K8sDeploymentEventCV; -import org.apache.streampark.flink.kubernetes.model.K8sEventKey; +import org.apache.streampark.flink.kubernetes.model.KubernetesDeploymentEventCV; +import org.apache.streampark.flink.kubernetes.model.KubernetesEventKey; import org.apache.flink.kubernetes.kubeclient.resources.CompKubernetesDeployment; import org.apache.flink.kubernetes.kubeclient.resources.CompatibleKubernetesWatcher; @@ -35,13 +35,13 @@ * would be tracked. The results of traced events would written into cachePool. */ @ThreadSafe -public class FlinkK8sEventWatcher extends FlinkWatcher { +public class FlinkKubernetesEventWatcher extends FlinkWatcher { - private final FlinkK8sWatchController watchController; + private final FlinkKubernetesWatchController watchController; private KubernetesClient k8sClient; - public FlinkK8sEventWatcher(FlinkK8sWatchController watchController) { + public FlinkKubernetesEventWatcher(FlinkKubernetesWatchController watchController) { this.watchController = watchController; } @@ -95,8 +95,8 @@ private void handleDeploymentEvent(Watcher.Action action, Deployment event) { String clusterId = event.getMetadata().getName(); String namespace = event.getMetadata().getNamespace(); watchController.k8sDeploymentEvents.put( - K8sEventKey.builder().namespace(namespace).clusterId(clusterId).build(), - K8sDeploymentEventCV.builder() + KubernetesEventKey.builder().namespace(namespace).clusterId(clusterId).build(), + KubernetesDeploymentEventCV.builder() .action(action) .event(event) .pollAckTime(System.currentTimeMillis()) diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkMetricWatcher.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkMetricWatcher.java index c04ecba5b3..04709afb46 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkMetricWatcher.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/watcher/FlinkMetricWatcher.java @@ -18,7 +18,7 @@ package org.apache.streampark.flink.kubernetes.watcher; import org.apache.streampark.flink.kubernetes.ChangeEventBus; -import org.apache.streampark.flink.kubernetes.FlinkK8sWatchController; +import org.apache.streampark.flink.kubernetes.FlinkKubernetesWatchController; import org.apache.streampark.flink.kubernetes.KubernetesRetriever; import org.apache.streampark.flink.kubernetes.MetricWatcherConfig; import org.apache.streampark.flink.kubernetes.event.FlinkClusterMetricChangeEvent; @@ -51,14 +51,14 @@ public class FlinkMetricWatcher extends FlinkWatcher { private final MetricWatcherConfig conf; - private final FlinkK8sWatchController watchController; + private final FlinkKubernetesWatchController watchController; private final ChangeEventBus eventBus; private ScheduledFuture timerSchedule; public FlinkMetricWatcher( MetricWatcherConfig conf, - FlinkK8sWatchController watchController, + FlinkKubernetesWatchController watchController, ChangeEventBus eventBus) { this.conf = conf; this.watchController = watchController; diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/spark/kubernetes/model/SparkK8sPodTemplates.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/spark/kubernetes/model/SparkKubernetesPodTemplates.java similarity index 84% rename from streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/spark/kubernetes/model/SparkK8sPodTemplates.java rename to streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/spark/kubernetes/model/SparkKubernetesPodTemplates.java index 9dcb11fbb6..af281b56d8 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/spark/kubernetes/model/SparkK8sPodTemplates.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/spark/kubernetes/model/SparkKubernetesPodTemplates.java @@ -25,7 +25,7 @@ /** Pod template for Spark k8s cluster */ @Builder @AllArgsConstructor -public class SparkK8sPodTemplates { +public class SparkKubernetesPodTemplates { @Builder.Default private final String driverPodTemplate = ""; @@ -48,12 +48,12 @@ public boolean isEmpty() { return !nonEmpty(); } - public static SparkK8sPodTemplates empty() { - return SparkK8sPodTemplates.builder().build(); + public static SparkKubernetesPodTemplates empty() { + return SparkKubernetesPodTemplates.builder().build(); } - public static SparkK8sPodTemplates of(String driverPodTemplate, String executorPodTemplate) { - return SparkK8sPodTemplates.builder() + public static SparkKubernetesPodTemplates of(String driverPodTemplate, String executorPodTemplate) { + return SparkKubernetesPodTemplates.builder() .driverPodTemplate(safeGet(driverPodTemplate)) .executorPodTemplate(safeGet(executorPodTemplate)) .build(); @@ -84,10 +84,10 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (!(obj instanceof SparkK8sPodTemplates)) { + if (!(obj instanceof SparkKubernetesPodTemplates)) { return false; } - SparkK8sPodTemplates that = (SparkK8sPodTemplates) obj; + SparkKubernetesPodTemplates that = (SparkKubernetesPodTemplates) obj; return trimSafe(driverPodTemplate).equals(trimSafe(that.driverPodTemplate)) && trimSafe(executorPodTemplate).equals(trimSafe(that.executorPodTemplate)); } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/PackerResourceGC.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/PackerResourceGC.java index 4bd4c1b4a1..d90a72d412 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/PackerResourceGC.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/PackerResourceGC.java @@ -17,8 +17,8 @@ package org.apache.streampark.flink.packer; -import org.apache.streampark.common.conf.Workspace; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.util.LoggerSupport; import org.apache.commons.io.FileUtils; @@ -33,7 +33,7 @@ public final class PackerResourceGC extends LoggerSupport { private static final PackerResourceGC INSTANCE = new PackerResourceGC(); - private static final String APP_WORKSPACE_PATH = Workspace.local().APP_WORKSPACE(); + private static final String APP_WORKSPACE_PATH = Workspace.LOCAL.workspace; private PackerResourceGC() { } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerRetriever.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerRetriever.java index ce18758957..4080c54959 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerRetriever.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/DockerRetriever.java @@ -17,8 +17,9 @@ package org.apache.streampark.flink.packer.docker; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.InternalConfigHolder; +import org.apache.streampark.common.configuration.Configuration; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.option.DockerOptions; import org.apache.streampark.common.util.Utils; import com.github.dockerjava.api.DockerClient; @@ -28,7 +29,6 @@ import com.github.dockerjava.httpclient5.ApacheDockerHttpClient; import java.net.URI; -import java.time.Duration; /** Docker client factory. */ public final class DockerRetriever { @@ -36,33 +36,23 @@ public final class DockerRetriever { public static final DockerClientConfig dockerClientConf = DefaultDockerClientConfig.createDefaultConfigBuilder().build(); - private static final ApacheDockerHttpClient.Builder dockerHttpClientBuilder = - new ApacheDockerHttpClient.Builder() - .dockerHost(dockerClientConf.getDockerHost()) - .sslConfig(dockerClientConf.getSSLConfig()) - .maxConnections(InternalConfigHolder.get(CommonConfig.DOCKER_MAX_CONNECTIONS())) - .connectionTimeout( - Duration.ofSeconds( - InternalConfigHolder.get(CommonConfig.DOCKER_CONNECTION_TIMEOUT_SEC()))) - .responseTimeout( - Duration.ofSeconds( - InternalConfigHolder.get(CommonConfig.DOCKER_RESPONSE_TIMEOUT_SEC()))); - private DockerRetriever() { } - /** get new DockerClient instance */ + /** Returns a new Docker client built from one consistent configuration snapshot. */ public static DockerClient newDockerClient() { - setDockerHost(); - return HackDockerClient.getInstance(dockerClientConf, dockerHttpClientBuilder.build()); - } - - /** set docker-host for kata */ - private static void setDockerHost() { - String dockerHost = InternalConfigHolder.get(CommonConfig.DOCKER_HOST()); + Configuration configuration = GlobalConfiguration.current(); + ApacheDockerHttpClient.Builder builder = + new ApacheDockerHttpClient.Builder() + .dockerHost(dockerClientConf.getDockerHost()) + .sslConfig(dockerClientConf.getSSLConfig()) + .maxConnections(configuration.get(DockerOptions.MAX_CONNECTIONS)) + .connectionTimeout(configuration.get(DockerOptions.CONNECTION_TIMEOUT)) + .responseTimeout(configuration.get(DockerOptions.RESPONSE_TIMEOUT)); + String dockerHost = configuration.get(DockerOptions.HOST); if (Utils.isNotEmpty(dockerHost)) { - URI dockerHostUri = URI.create(dockerHost); - dockerHttpClientBuilder.dockerHost(dockerHostUri); + builder.dockerHost(URI.create(dockerHost)); } + return HackDockerClient.getInstance(dockerClientConf, builder.build()); } } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplateTrait.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplateTrait.java index a10509b153..b6ec08f6c5 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplateTrait.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/FlinkDockerfileTemplateTrait.java @@ -17,7 +17,7 @@ package org.apache.streampark.flink.packer.docker; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; import org.apache.streampark.common.fs.LfsOperator; import org.apache.commons.io.FileUtils; diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplateTrait.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplateTrait.java index f311d4e416..0ffbe25a67 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplateTrait.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/docker/SparkDockerfileTemplateTrait.java @@ -17,7 +17,7 @@ package org.apache.streampark.flink.packer.docker; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Constants; import org.apache.streampark.common.fs.LfsOperator; import org.apache.commons.io.FileUtils; diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/MavenTool.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/MavenTool.java index 3407edb73e..888a8081ef 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/MavenTool.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/maven/MavenTool.java @@ -17,10 +17,11 @@ package org.apache.streampark.flink.packer.maven; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.InternalConfigHolder; -import org.apache.streampark.common.conf.Workspace; -import org.apache.streampark.common.constants.Constants; +import org.apache.streampark.common.configuration.Configuration; +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.Workspace; +import org.apache.streampark.common.configuration.option.MavenOptions; import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.LoggerSupport; import org.apache.streampark.common.util.Utils; @@ -79,18 +80,18 @@ private MavenTool() { } private static List getRemoteRepos() { + Configuration configuration = GlobalConfiguration.current(); RemoteRepository.Builder builder = new RemoteRepository.Builder( "central", Constants.DEFAULT, - InternalConfigHolder.get(CommonConfig.MAVEN_REMOTE_URL())); + configuration.get(MavenOptions.REPOSITORY_URL)); RemoteRepository remoteRepository; - if (InternalConfigHolder.get(CommonConfig.MAVEN_AUTH_USER()) == null - || InternalConfigHolder.get(CommonConfig.MAVEN_AUTH_PASSWORD()) == null) { + String username = configuration.getOptional(MavenOptions.USER_NAME).orElse(null); + String password = configuration.getOptional(MavenOptions.PASSWORD).orElse(null); + if (username == null || password == null) { remoteRepository = builder.build(); } else { - String username = InternalConfigHolder.get(CommonConfig.MAVEN_AUTH_USER()); - String password = InternalConfigHolder.get(CommonConfig.MAVEN_AUTH_PASSWORD()); org.eclipse.aether.repository.Authentication authentication = new AuthenticationBuilder() .addUsername(username) @@ -268,7 +269,7 @@ public static List resolveArtifacts(@Nullable Set mavenArtifacts private static MavenEndpoint getMavenEndpoint() { org.eclipse.aether.impl.DefaultServiceLocator locator = MavenRepositorySystemUtils.newServiceLocator(); - LocalRepository localRepo = new LocalRepository(Workspace.MAVEN_LOCAL_PATH()); + LocalRepository localRepo = new LocalRepository(Workspace.MAVEN_LOCAL_PATH); locator.addService( RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class); diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkBuildParam.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkBuildParam.java index b86a61ead4..d1ddfb8bdc 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkBuildParam.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkBuildParam.java @@ -17,8 +17,8 @@ package org.apache.streampark.flink.packer.pipeline; -import org.apache.streampark.common.conf.FlinkVersion; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.Workspace; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; import org.apache.streampark.flink.packer.maven.DependencyInfo; @@ -43,11 +43,11 @@ public interface FlinkBuildParam extends BuildParam { default DependencyInfo providedLibs() { Set libs = new HashSet<>(Arrays.asList( - Workspace.local().APP_JARS(), + Workspace.LOCAL.jars, customFlinkUserJar())); if (flinkJobType() == FlinkJobType.FLINK_SQL) { libs.add( - Workspace.local().APP_SHIMS() + Workspace.LOCAL.shims + "/flink-" + flinkVersion().majorVersion()); } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sApplicationBuildRequest.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sApplicationBuildRequest.java index b24eb69581..ea23178f41 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sApplicationBuildRequest.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sApplicationBuildRequest.java @@ -17,10 +17,10 @@ package org.apache.streampark.flink.packer.pipeline; -import org.apache.streampark.common.conf.FlinkVersion; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; -import org.apache.streampark.flink.kubernetes.model.K8sPodTemplates; +import org.apache.streampark.flink.kubernetes.model.KubernetesPodTemplates; import org.apache.streampark.flink.packer.docker.DockerConf; import org.apache.streampark.flink.packer.maven.DependencyInfo; @@ -37,7 +37,7 @@ public class FlinkK8sApplicationBuildRequest implements FlinkK8sBuildParam { private final String clusterId; private final String k8sNamespace; private final String flinkBaseImage; - private final K8sPodTemplates flinkPodTemplate; + private final KubernetesPodTemplates flinkPodTemplate; private final boolean integrateWithHadoop; private final DockerConf dockerConfig; private final String ingressTemplate; @@ -54,7 +54,7 @@ public FlinkK8sApplicationBuildRequest( String clusterId, String k8sNamespace, String flinkBaseImage, - K8sPodTemplates flinkPodTemplate, + KubernetesPodTemplates flinkPodTemplate, boolean integrateWithHadoop, DockerConf dockerConfig, String ingressTemplate) { @@ -129,7 +129,7 @@ public String flinkBaseImage() { return flinkBaseImage; } - public K8sPodTemplates flinkPodTemplate() { + public KubernetesPodTemplates flinkPodTemplate() { return flinkPodTemplate; } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sSessionBuildRequest.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sSessionBuildRequest.java index fd14dd84f7..dd8f8bff5c 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sSessionBuildRequest.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkK8sSessionBuildRequest.java @@ -17,7 +17,7 @@ package org.apache.streampark.flink.packer.pipeline; -import org.apache.streampark.common.conf.FlinkVersion; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; import org.apache.streampark.flink.packer.maven.DependencyInfo; diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkRemotePerJobBuildRequest.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkRemotePerJobBuildRequest.java index 32922a72b8..eebbfc6e0a 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkRemotePerJobBuildRequest.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/FlinkRemotePerJobBuildRequest.java @@ -17,7 +17,7 @@ package org.apache.streampark.flink.packer.pipeline; -import org.apache.streampark.common.conf.FlinkVersion; +import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; import org.apache.streampark.flink.packer.maven.DependencyInfo; diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SparkK8sApplicationBuildRequest.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SparkK8sApplicationBuildRequest.java index 494026be94..6845844777 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SparkK8sApplicationBuildRequest.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/SparkK8sApplicationBuildRequest.java @@ -17,12 +17,12 @@ package org.apache.streampark.flink.packer.pipeline; -import org.apache.streampark.common.conf.SparkVersion; +import org.apache.streampark.common.core.SparkVersion; import org.apache.streampark.common.enums.SparkDeployMode; import org.apache.streampark.common.enums.SparkJobType; import org.apache.streampark.flink.packer.docker.DockerConf; import org.apache.streampark.flink.packer.maven.DependencyInfo; -import org.apache.streampark.spark.kubernetes.model.SparkK8sPodTemplates; +import org.apache.streampark.spark.kubernetes.model.SparkKubernetesPodTemplates; public class SparkK8sApplicationBuildRequest implements BuildParam { @@ -36,7 +36,7 @@ public class SparkK8sApplicationBuildRequest implements BuildParam { private final DependencyInfo dependencyInfo; private final String k8sNamespace; private final String sparkBaseImage; - private final SparkK8sPodTemplates sparkPodTemplate; + private final SparkKubernetesPodTemplates sparkPodTemplate; private final boolean integrateWithHadoop; private final DockerConf dockerConfig; @@ -51,7 +51,7 @@ public SparkK8sApplicationBuildRequest( DependencyInfo dependencyInfo, String k8sNamespace, String sparkBaseImage, - SparkK8sPodTemplates sparkPodTemplate, + SparkKubernetesPodTemplates sparkPodTemplate, boolean integrateWithHadoop, DockerConf dockerConfig) { this.appName = appName; @@ -111,7 +111,7 @@ public String sparkBaseImage() { return sparkBaseImage; } - public SparkK8sPodTemplates sparkPodTemplate() { + public SparkKubernetesPodTemplates sparkPodTemplate() { return sparkPodTemplate; } diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/YarnJarUploader.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/YarnJarUploader.java index cb524119bf..773c7bcf2b 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/YarnJarUploader.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/YarnJarUploader.java @@ -17,7 +17,7 @@ package org.apache.streampark.flink.packer.pipeline; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.common.util.AutoCloseUtils; @@ -50,7 +50,7 @@ private static void uploadFile(FsOperator fsOperator, File originFile, String ta fsOperator.copy(originFile.getAbsolutePath(), target); return; } - String uploadFile = Workspace.remote().APP_UPLOADS() + "/" + originFile.getName(); + String uploadFile = Workspace.REMOTE.uploads + "/" + originFile.getName(); if (fsOperator.exists(uploadFile)) { AutoCloseUtils.using( new FileInputStream(originFile), diff --git a/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/DockerClientTest.java b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/DockerClientTest.java index 743c073ca8..040e6003d7 100644 --- a/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/DockerClientTest.java +++ b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/DockerClientTest.java @@ -17,8 +17,8 @@ package org.apache.streampark.flink.packer; -import org.apache.streampark.common.conf.CommonConfig; -import org.apache.streampark.common.conf.InternalConfigHolder; +import org.apache.streampark.common.configuration.GlobalConfiguration; +import org.apache.streampark.common.configuration.option.DockerOptions; import org.apache.streampark.flink.packer.docker.DockerImageExist; import org.apache.streampark.flink.packer.docker.DockerRetriever; import org.apache.streampark.flink.packer.docker.DockerUtils; @@ -32,7 +32,6 @@ import java.nio.file.Files; import java.nio.file.Paths; -import java.time.Duration; import java.util.concurrent.TimeUnit; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -86,13 +85,11 @@ void dockerClientConfigBuilder() { new ApacheDockerHttpClient.Builder() .dockerHost(DockerRetriever.dockerClientConf.getDockerHost()) .sslConfig(DockerRetriever.dockerClientConf.getSSLConfig()) - .maxConnections(InternalConfigHolder.get(CommonConfig.DOCKER_MAX_CONNECTIONS())) + .maxConnections(GlobalConfiguration.current().get(DockerOptions.MAX_CONNECTIONS)) .connectionTimeout( - Duration.ofSeconds( - InternalConfigHolder.get(CommonConfig.DOCKER_CONNECTION_TIMEOUT_SEC()))) + GlobalConfiguration.current().get(DockerOptions.CONNECTION_TIMEOUT)) .responseTimeout( - Duration.ofSeconds( - InternalConfigHolder.get(CommonConfig.DOCKER_RESPONSE_TIMEOUT_SEC()))); + GlobalConfiguration.current().get(DockerOptions.RESPONSE_TIMEOUT)); assertNotNull(dockerHttpClientBuilder); } diff --git a/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/MavenToolTest.java b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/MavenToolTest.java index a1922c2264..e783154bfb 100644 --- a/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/MavenToolTest.java +++ b/streampark-flink/streampark-flink-packer/src/test/java/org/apache/streampark/flink/packer/MavenToolTest.java @@ -17,7 +17,7 @@ package org.apache.streampark.flink.packer; -import org.apache.streampark.common.conf.Workspace; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.flink.packer.maven.Artifact; import org.apache.streampark.flink.packer.maven.DependencyInfo; import org.apache.streampark.flink.packer.maven.MavenTool; @@ -52,7 +52,7 @@ class MavenToolTest { @BeforeAll static void beforeAll() throws Exception { - FileUtils.forceMkdir(new File(Workspace.MAVEN_LOCAL_PATH())); + FileUtils.forceMkdir(new File(Workspace.MAVEN_LOCAL_PATH)); File output = new File(OUTPUT_DIR); FileUtils.deleteDirectory(output); FileUtils.forceMkdir(output); diff --git a/streampark-flink/streampark-flink-proxy/pom.xml b/streampark-flink/streampark-flink-proxy/pom.xml deleted file mode 100644 index b056e1b240..0000000000 --- a/streampark-flink/streampark-flink-proxy/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink - ${revision} - - - streampark-flink-proxy - StreamPark : Flink Proxy - - - - - org.apache.streampark - streampark-common - provided - - - - - - - apache-release - - true - - - - - diff --git a/streampark-flink/streampark-flink-proxy/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java b/streampark-flink/streampark-flink-proxy/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java deleted file mode 100644 index c16d1ee869..0000000000 --- a/streampark-flink/streampark-flink-proxy/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.proxy; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.io.Serializable; - -class FlinkShimsProxyTest { - - @Test - void getObjectShouldDeserializeAcrossClassLoaders() throws Exception { - String payload = "streampark-proxy-test"; - ClassLoader loader = Thread.currentThread().getContextClassLoader(); - Serializable roundTripped = - FlinkShimsProxy.getObject(loader, payload, String.class); - Assertions.assertEquals(payload, roundTripped); - } - - @Test - void getObjectShouldPreserveCustomSerializableType() throws Exception { - TestBean original = new TestBean("app-1", 42); - ClassLoader loader = Thread.currentThread().getContextClassLoader(); - TestBean roundTripped = FlinkShimsProxy.getObject(loader, original, TestBean.class); - Assertions.assertEquals(original, roundTripped); - } - - private static final class TestBean implements Serializable { - - private static final long serialVersionUID = 1L; - private final String name; - private final int value; - - TestBean(String name, int value) { - this.name = name; - this.value = value; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof TestBean)) { - return false; - } - TestBean testBean = (TestBean) o; - return value == testBean.value && name.equals(testBean.name); - } - - @Override - public int hashCode() { - return 31 * name.hashCode() + value; - } - } -} diff --git a/streampark-flink/streampark-flink-shims/pom.xml b/streampark-flink/streampark-flink-shims/pom.xml index 42282c4634..cbac25723d 100644 --- a/streampark-flink/streampark-flink-shims/pom.xml +++ b/streampark-flink/streampark-flink-shims/pom.xml @@ -31,28 +31,13 @@ streampark-flink-shims-base - streampark-flink-shims-base-v2 - streampark-flink-shims-test - streampark-flink-shims_flink-1.17 streampark-flink-shims_flink-1.18 streampark-flink-shims_flink-1.19 streampark-flink-shims_flink-1.20 + streampark-flink-shims_flink-2.0 + streampark-flink-shims_flink-2.1 + streampark-flink-shims_flink-2.2 + streampark-flink-shims_flink-2.3 - - - - flink-2.x-shims - - [11,) - - - streampark-flink-shims_flink-2.0 - streampark-flink-shims_flink-2.1 - streampark-flink-shims_flink-2.2 - streampark-flink-shims_flink-2.3 - - - - diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/pom.xml deleted file mode 100644 index 3d4af1042d..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/pom.xml +++ /dev/null @@ -1,109 +0,0 @@ - - - - 4.0.0 - - - org.apache.streampark - streampark-flink-shims - ${revision} - - - streampark-flink-shims-base-v2 - StreamPark : Flink Shims Base V2 - - - 2.0.2 - - - - - org.apache.streampark - streampark-flink-shims-base - ${project.version} - - - - org.apache.streampark - streampark-common - - - - org.apache.flink - flink-core - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-java - ${flink.version} - provided - - - - org.apache.flink - flink-clients - ${flink.version} - provided - - - - org.apache.flink - flink-kubernetes - ${flink.version} - provided - - - - org.apache.flink - flink-table-api-java - ${flink.version} - provided - - - - org.apache.flink - flink-table-api-java-bridge - ${flink.version} - provided - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.hadoop - hadoop-client-api - true - - - - org.apache.hadoop - hadoop-client-runtime - true - - - - diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java deleted file mode 100644 index eae3584f58..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.util.AssertUtils; -import org.apache.streampark.common.util.StreamParkLoggerFactory; - -import org.apache.streampark.shaded.org.slf4j.Logger; - -import org.apache.commons.lang3.StringUtils; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.configuration.ExecutionOptions; -import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.TableResult; -import org.apache.flink.table.catalog.ResolvedSchema; -import org.apache.flink.types.Row; -import org.apache.flink.util.ParameterTool; - -import java.util.Arrays; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -/** Executes Flink SQL statements. */ -public final class FlinkSqlExecutor { - - private static final Logger LOG = - StreamParkLoggerFactory.loggerFactory() - .getLogger(FlinkSqlExecutor.class.getName()); - - private static final ReentrantReadWriteLock.WriteLock LOCK = - new ReentrantReadWriteLock().writeLock(); - - private static final Map COMMAND_HANDLERS = buildCommandHandlers(); - - private FlinkSqlExecutor() { - } - - public static void executeSql(String sql, ParameterTool parameter, TableEnvironment context) { - executeSql(sql, parameter, context, null); - } - - public static void executeSql( - String sql, - ParameterTool parameter, - TableEnvironment context, - Consumer callbackFunc) { - String flinkSql = resolveSql(sql, parameter); - ExecutionContext ctx = new ExecutionContext(context, callbackFunc, parameter); - List calls = SqlCommandParser.parseSQL(flinkSql, null); - for (SqlCommandCall call : calls) { - processCommand(call, ctx); - } - finishExecution(flinkSql, ctx); - } - - private static String resolveSql(String sql, ParameterTool parameter) { - String flinkSql = - StringUtils.isBlank(sql) - ? parameter.get(ConfigKeys.KEY_FLINK_SQL()) - : parameter.get(sql); - if (StringUtils.isBlank(flinkSql)) { - throw new IllegalArgumentException("verify failed: flink sql cannot be empty"); - } - return flinkSql; - } - - private static void processCommand(SqlCommandCall call, ExecutionContext ctx) { - COMMAND_HANDLERS.getOrDefault(call.command, FlinkSqlExecutor::executeDefault).handle(call, ctx); - } - - private static void finishExecution(String flinkSql, ExecutionContext ctx) { - if (ctx.hasInsert) { - TableResult result = ctx.statementSet.execute(); - if (result != null) { - result.getJobClient() - .ifPresent( - jobClient -> { - try { - LOG.info("jobId:{}", jobClient.getJobID()); - } catch (Exception ignored) { - // ignore - } - }); - } - } else { - LOG.error("No 'INSERT' statement to trigger the execution of the Flink job."); - throw new IllegalStateException( - "No 'INSERT' statement to trigger the execution of the Flink job."); - } - - LOG.info( - "\n\n\n==============flinkSql==============\n\n {}\n\n============================\n\n\n", - flinkSql); - } - - private static Map buildCommandHandlers() { - Map handlers = new EnumMap<>(SqlCommand.class); - handlers.put(SqlCommand.SHOW_CATALOGS, - (call, ctx) -> showMeta(call, ctx, joinLines(ctx.context.listCatalogs()))); - handlers.put( - SqlCommand.SHOW_CURRENT_CATALOG, - (call, ctx) -> showMeta(call, ctx, ctx.context.getCurrentCatalog())); - handlers.put( - SqlCommand.SHOW_DATABASES, - (call, ctx) -> showMeta(call, ctx, joinLines(ctx.context.listDatabases()))); - handlers.put( - SqlCommand.SHOW_CURRENT_DATABASE, - (call, ctx) -> showMeta(call, ctx, ctx.context.getCurrentDatabase())); - handlers.put(SqlCommand.SHOW_TABLES, FlinkSqlExecutor::showTables); - handlers.put( - SqlCommand.SHOW_FUNCTIONS, - (call, ctx) -> showMeta(call, ctx, joinLines(ctx.context.listUserDefinedFunctions()))); - handlers.put( - SqlCommand.SHOW_MODULES, - (call, ctx) -> showMeta(call, ctx, joinLines(ctx.context.listModules()))); - handlers.put(SqlCommand.DESC, FlinkSqlExecutor::describeTable); - handlers.put(SqlCommand.DESCRIBE, FlinkSqlExecutor::describeTable); - handlers.put(SqlCommand.EXPLAIN, FlinkSqlExecutor::explainSql); - handlers.put(SqlCommand.SET, FlinkSqlExecutor::setConfig); - handlers.put(SqlCommand.RESET, FlinkSqlExecutor::resetConfig); - handlers.put(SqlCommand.RESET_ALL, FlinkSqlExecutor::resetConfig); - handlers.put(SqlCommand.BEGIN_STATEMENT_SET, FlinkSqlExecutor::warnStatementSet); - handlers.put(SqlCommand.END_STATEMENT_SET, FlinkSqlExecutor::warnStatementSet); - handlers.put(SqlCommand.INSERT, FlinkSqlExecutor::addInsert); - handlers.put(SqlCommand.SELECT, FlinkSqlExecutor::rejectSelect); - handlers.put(SqlCommand.DELETE, FlinkSqlExecutor::validateBatchCommand); - handlers.put(SqlCommand.UPDATE, FlinkSqlExecutor::validateBatchCommand); - return handlers; - } - - private static void showMeta(SqlCommandCall call, ExecutionContext ctx, String payload) { - ctx.callback.accept(call.command.getName() + ": " + payload); - } - - private static void showTables(SqlCommandCall call, ExecutionContext ctx) { - String tables = - Arrays.stream(ctx.context.listTables()) - .filter(t -> !t.startsWith("UnnamedTable")) - .collect(Collectors.joining("\n")); - showMeta(call, ctx, tables); - } - - private static void describeTable(SqlCommandCall call, ExecutionContext ctx) { - String args = firstOperand(call); - ResolvedSchema schema = ctx.context.from(args).getResolvedSchema(); - StringBuilder builder = new StringBuilder(); - builder.append("Column\tType\n"); - for (int i = 0; i < schema.getColumnCount(); i++) { - builder.append(schema.getColumnNames().get(i)) - .append("\t") - .append(schema.getColumnDataTypes().get(i)) - .append("\n"); - } - ctx.callback.accept(builder.toString()); - } - - private static void explainSql(SqlCommandCall call, ExecutionContext ctx) { - TableResult tableResult = ctx.context.executeSql(call.originSql); - Row row = tableResult.collect().next(); - ctx.callback.accept(row.getField(0).toString()); - } - - private static void setConfig(SqlCommandCall call, ExecutionContext ctx) { - AssertUtils.required( - call.operands != null && call.operands.length >= 2, - "SET command requires key and value operands"); - String args = call.operands[0]; - String operand = call.operands[1]; - LOG.info("{}: {} --> {}", call.command.getName(), args, operand); - ctx.context.getConfig().getConfiguration().setString(args, operand); - } - - private static void resetConfig(SqlCommandCall call, ExecutionContext ctx) { - String args = firstOperand(call); - try { - java.lang.reflect.Field confDataField = - Configuration.class.getDeclaredField("confData"); - confDataField.setAccessible(true); - @SuppressWarnings("unchecked") - HashMap confData = - (HashMap) confDataField.get(ctx.context.getConfig().getConfiguration()); - synchronized (confData) { - if (call.command == SqlCommand.RESET) { - confData.remove(args); - } else { - confData.clear(); - } - } - LOG.info("{}: {}", call.command.getName(), args); - } catch (ReflectiveOperationException e) { - throw new IllegalStateException("Failed to reset Flink table configuration", e); - } - } - - private static void warnStatementSet(SqlCommandCall call, ExecutionContext ctx) { - LOG.warn("SQL Client Syntax: {} ", call.command.getName()); - } - - private static void addInsert(SqlCommandCall call, ExecutionContext ctx) { - ctx.statementSet.addInsertSql(call.originSql); - ctx.hasInsert = true; - } - - private static void rejectSelect(SqlCommandCall call, ExecutionContext ctx) { - LOG.error("StreamPark dose not support 'SELECT' statement now!"); - throw new UnsupportedOperationException("StreamPark dose not support 'select' statement now!"); - } - - private static void validateBatchCommand(SqlCommandCall call, ExecutionContext ctx) { - String runMode = ctx.parameter.get(ExecutionOptions.RUNTIME_MODE.key()); - AssertUtils.required( - !"STREAMING".equals(runMode), - "Currently, " - + call.command.getName().toUpperCase() - + " statement only supports in batch mode, " - + "and it requires the target table connector implements the SupportsRowLevelDelete, " - + "For more details please refer to: https://nightlies.apache.org/flink/flink-docs-release-1.18/docs/dev/table/sql/" - + call.command.getName()); - } - - private static void executeDefault(SqlCommandCall call, ExecutionContext ctx) { - String args = firstOperand(call); - try { - LOCK.lock(); - ctx.context.executeSql(call.originSql); - LOG.info("{}:{}", call.command.getName(), args); - } finally { - if (LOCK.isHeldByCurrentThread()) { - LOCK.unlock(); - } - } - } - - private static String firstOperand(SqlCommandCall call) { - return call.operands.length == 0 ? null : call.operands[0]; - } - - private static String joinLines(String[] values) { - return String.join("\n", values); - } - - @FunctionalInterface - private interface CommandHandler { - - void handle(SqlCommandCall call, ExecutionContext ctx); - } - - private static final class ExecutionContext { - - private final TableEnvironment context; - private final Consumer callback; - private final ParameterTool parameter; - private final org.apache.flink.table.api.StatementSet statementSet; - private boolean hasInsert; - - private ExecutionContext( - TableEnvironment context, Consumer callbackFunc, - ParameterTool parameter) { - this.context = context; - this.parameter = parameter; - this.statementSet = context.createStatementSet(); - this.callback = - r -> { - if (callbackFunc != null) { - callbackFunc.accept(r); - } else { - LOG.info(r); - } - }; - } - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTraitV2.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTraitV2.java deleted file mode 100644 index 63b5c2d39e..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTraitV2.java +++ /dev/null @@ -1,691 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.streampark.common.util.Utils; - -import org.apache.flink.api.common.JobExecutionResult; -import org.apache.flink.api.common.RuntimeExecutionMode; -import org.apache.flink.api.common.cache.DistributedCache; -import org.apache.flink.api.common.eventtime.WatermarkStrategy; -import org.apache.flink.api.common.io.FileInputFormat; -import org.apache.flink.api.common.io.InputFormat; -import org.apache.flink.api.connector.source.Source; -import org.apache.flink.api.connector.source.SourceSplit; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.configuration.ReadableConfig; -import org.apache.flink.core.execution.JobClient; -import org.apache.flink.core.execution.JobListener; -import org.apache.flink.streaming.api.CheckpointingMode; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.CheckpointConfig; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.FileProcessingMode; -import org.apache.flink.streaming.api.graph.StreamGraph; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.Schema; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableConfig; -import org.apache.flink.table.api.TableDescriptor; -import org.apache.flink.table.api.TableException; -import org.apache.flink.table.api.TableResult; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.table.catalog.Catalog; -import org.apache.flink.table.catalog.CatalogDescriptor; -import org.apache.flink.table.connector.ChangelogMode; -import org.apache.flink.table.expressions.Expression; -import org.apache.flink.table.functions.ScalarFunction; -import org.apache.flink.table.functions.UserDefinedFunction; -import org.apache.flink.table.module.Module; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; -import org.apache.flink.table.types.AbstractDataType; -import org.apache.flink.types.Row; -import org.apache.flink.util.ParameterTool; -import org.apache.flink.util.SplittableIterator; - -import java.util.Collection; -import java.util.List; -import java.util.Optional; -import java.util.function.Consumer; - -/** - * Integration API of stream and table environments for Flink 2.x. - * - *

Once a Table has been converted to a DataStream, the DataStream job must be executed using the - * execute method of the StreamExecutionEnvironment. - */ -public abstract class FlinkStreamTableTraitV2 implements StreamTableEnvironment { - - public final ParameterTool parameter; - - private final StreamExecutionEnvironment streamEnv; - - private final StreamTableEnvironment tableEnv; - - /** Whether a table has been converted to a DataStream. */ - public boolean isConvertedToDataStream; - - protected FlinkStreamTableTraitV2( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment tableEnv) { - this.parameter = parameter; - this.streamEnv = streamEnv; - this.tableEnv = tableEnv; - } - - protected StreamExecutionEnvironment getStreamEnv() { - return streamEnv; - } - - protected StreamTableEnvironment getStreamTableEnv() { - return tableEnv; - } - - /** Recommended API to start tasks. */ - public JobExecutionResult start() { - return start(null); - } - - public JobExecutionResult start(String name) { - String appName = FlinkParameterUtils.getAppName(parameter, name, true); - return execute(appName); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - public JobExecutionResult execute(String jobName) { - Utils.printLogo("FlinkStreamTable " + jobName + " Starting..."); - if (isConvertedToDataStream) { - try { - return streamEnv.execute(jobName); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - return null; - } - - public void sql(String sql) { - sql(sql, null); - } - - public void sql(String sql, Consumer callback) { - FlinkSqlExecutor.executeSql(sql, parameter, this, callback); - } - - public StreamExecutionEnvironment getJavaEnv() { - return streamEnv; - } - - public List> $getCachedFiles() { - return streamEnv.getCachedFiles(); - } - - public List $getJobListeners() { - return streamEnv.getJobListeners(); - } - - public void $setParallelism(int parallelism) { - streamEnv.setParallelism(parallelism); - } - - public StreamExecutionEnvironment $setRuntimeMode(RuntimeExecutionMode deployMode) { - return streamEnv.setRuntimeMode(deployMode); - } - - public void $setMaxParallelism(int maxParallelism) { - streamEnv.setMaxParallelism(maxParallelism); - } - - public int $getParallelism() { - return streamEnv.getParallelism(); - } - - public int $getMaxParallelism() { - return streamEnv.getMaxParallelism(); - } - - public StreamExecutionEnvironment $setBufferTimeout(long timeoutMillis) { - return streamEnv.setBufferTimeout(timeoutMillis); - } - - public long $getBufferTimeout() { - return streamEnv.getBufferTimeout(); - } - - public StreamExecutionEnvironment $disableOperatorChaining() { - return streamEnv.disableOperatorChaining(); - } - - public CheckpointConfig $getCheckpointConfig() { - return streamEnv.getCheckpointConfig(); - } - - public StreamExecutionEnvironment $enableCheckpointing(long interval, CheckpointingMode mode) { - return streamEnv.enableCheckpointing(interval, mode); - } - - public StreamExecutionEnvironment $enableCheckpointing(long interval) { - return streamEnv.enableCheckpointing(interval); - } - - public CheckpointingMode $getCheckpointingMode() { - return streamEnv.getCheckpointingMode(); - } - - public void $configure(ReadableConfig configuration, ClassLoader classLoader) { - streamEnv.configure(configuration, classLoader); - } - - public DataStream $fromData(T data) { - return streamEnv.fromData(data); - } - - public DataStream $fromSequence(long from, long to) { - return streamEnv.fromSequence(from, to); - } - - public DataStream $fromCollection(Collection data) { - return streamEnv.fromCollection(data); - } - - public DataStream $fromParallelCollection(SplittableIterator data, Class clazz) { - return streamEnv.fromParallelCollection(data, clazz); - } - - public DataStream $readFile(FileInputFormat inputFormat, String filePath) { - return streamEnv.readFile(inputFormat, filePath); - } - - public DataStream $readFile( - FileInputFormat inputFormat, - String filePath, - FileProcessingMode watchType, - long interval) { - return streamEnv.readFile(inputFormat, filePath, watchType, interval); - } - - public DataStream $socketTextStream( - String hostname, int port, char delimiter, long maxRetry) { - return streamEnv.socketTextStream(hostname, port, delimiter, maxRetry); - } - - public DataStream $createInput(InputFormat inputFormat) { - return streamEnv.createInput(inputFormat); - } - - public DataStream $fromSource( - Source source, - WatermarkStrategy watermarkStrategy, - String sourceName) { - return streamEnv.fromSource(source, watermarkStrategy, sourceName); - } - - public void $registerJobListener(JobListener jobListener) { - streamEnv.registerJobListener(jobListener); - } - - public void $clearJobListeners() { - streamEnv.clearJobListeners(); - } - - public JobClient $executeAsync() { - try { - return streamEnv.executeAsync(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public JobClient $executeAsync(String jobName) { - try { - return streamEnv.executeAsync(jobName); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public String $getExecutionPlan() { - return streamEnv.getExecutionPlan(); - } - - public StreamGraph $getStreamGraph() { - return streamEnv.getStreamGraph(); - } - - public StreamExecutionEnvironment $getWrappedStreamExecutionEnvironment() { - return streamEnv; - } - - public void $registerCachedFile(String filePath, String name) { - streamEnv.registerCachedFile(filePath, name); - } - - public void $registerCachedFile(String filePath, String name, boolean executable) { - streamEnv.registerCachedFile(filePath, name, executable); - } - - public boolean $isUnalignedCheckpointsEnabled() { - return streamEnv.isUnalignedCheckpointsEnabled(); - } - - public boolean $isForceUnalignedCheckpoints() { - return streamEnv.isForceUnalignedCheckpoints(); - } - - @Override - public Table fromDataStream(DataStream dataStream, Schema schema) { - return tableEnv.fromDataStream(dataStream, schema); - } - - @Override - public Table fromChangelogStream(DataStream dataStream) { - return tableEnv.fromChangelogStream(dataStream); - } - - @Override - public Table fromChangelogStream(DataStream dataStream, Schema schema) { - return tableEnv.fromChangelogStream(dataStream, schema); - } - - @Override - public Table fromChangelogStream( - DataStream dataStream, Schema schema, ChangelogMode changelogMode) { - return tableEnv.fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView(String path, DataStream dataStream, Schema schema) { - tableEnv.createTemporaryView(path, dataStream, schema); - } - - @Override - public DataStream toDataStream(Table table) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table); - } - - @Override - public DataStream toDataStream(Table table, Class targetClass) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetClass); - } - - @Override - public DataStream toDataStream(Table table, AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetDataType); - } - - @Override - public DataStream toChangelogStream(Table table) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table); - } - - @Override - public DataStream toChangelogStream(Table table, Schema targetSchema) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema); - } - - @Override - public DataStream toChangelogStream( - Table table, Schema targetSchema, ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public DataStream toAppendStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return tableEnv.toAppendStream(table, clazz); - } - - @Override - public DataStream> toRetractStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return tableEnv.toRetractStream(table, clazz); - } - - @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - tableEnv.createCatalog(catalogName, catalogDescriptor); - } - - @Override - public void useModules(String... moduleNames) { - tableEnv.useModules(moduleNames); - } - - @Override - public void createFunction( - String path, String className, List resourceUris) { - tableEnv.createFunction(path, className, resourceUris); - } - - @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - tableEnv.createFunction(path, className, resourceUris, ignoreIfExists); - } - - @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - tableEnv.createTemporaryFunction(path, className, resourceUris); - } - - @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - tableEnv.createTemporarySystemFunction(name, className, resourceUris); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - tableEnv.createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - tableEnv.createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor descriptor) { - return tableEnv.from(descriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); - } - - @Override - public String[] listTables(String catalogName, String databaseName) { - return tableEnv.listTables(catalogName, databaseName); - } - - @Override - public String explainSql( - String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, format, extraDetails); - } - - @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return tableEnv.loadPlan(planReference); - } - - @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return tableEnv.compilePlanSql(statement); - } - - @Override - public Table fromDataStream(DataStream dataStream) { - return tableEnv.fromDataStream(dataStream); - } - - @Override - public Table fromDataStream(DataStream dataStream, Expression... fields) { - return tableEnv.fromDataStream(dataStream, fields); - } - - @Override - public void createTemporaryView(String path, DataStream dataStream) { - tableEnv.createTemporaryView(path, dataStream); - } - - @Override - public void createTemporaryView( - String path, DataStream dataStream, Expression... fields) { - tableEnv.createTemporaryView(path, dataStream, fields); - } - - @Override - public Table fromValues(Expression... values) { - return tableEnv.fromValues(values); - } - - @Override - public Table fromValues(AbstractDataType rowType, Expression... values) { - return tableEnv.fromValues(rowType, values); - } - - @Override - public Table fromValues(Iterable values) { - return tableEnv.fromValues(values); - } - - @Override - public Table fromValues(AbstractDataType rowType, Iterable values) { - return tableEnv.fromValues(rowType, values); - } - - @Override - public void registerCatalog(String catalogName, Catalog catalog) { - tableEnv.registerCatalog(catalogName, catalog); - } - - @Override - public Optional getCatalog(String catalogName) { - return tableEnv.getCatalog(catalogName); - } - - @Override - public void loadModule(String moduleName, Module module) { - tableEnv.loadModule(moduleName, module); - } - - @Override - public void unloadModule(String moduleName) { - tableEnv.unloadModule(moduleName); - } - - @Override - public void createTemporarySystemFunction( - String name, Class functionClass) { - tableEnv.createTemporarySystemFunction(name, functionClass); - } - - @Override - public void createTemporarySystemFunction( - String name, UserDefinedFunction functionInstance) { - tableEnv.createTemporarySystemFunction(name, functionInstance); - } - - @Override - public boolean dropTemporarySystemFunction(String name) { - return tableEnv.dropTemporarySystemFunction(name); - } - - @Override - public void createFunction(String path, Class functionClass) { - tableEnv.createFunction(path, functionClass); - } - - @Override - public void createFunction( - String path, - Class functionClass, - boolean ignoreIfExists) { - tableEnv.createFunction(path, functionClass, ignoreIfExists); - } - - @Override - public boolean dropFunction(String path) { - return tableEnv.dropFunction(path); - } - - @Override - public void createTemporaryFunction( - String path, Class functionClass) { - tableEnv.createTemporaryFunction(path, functionClass); - } - - @Override - public void createTemporaryFunction(String path, UserDefinedFunction functionInstance) { - tableEnv.createTemporaryFunction(path, functionInstance); - } - - @Override - public boolean dropTemporaryFunction(String path) { - return tableEnv.dropTemporaryFunction(path); - } - - @Override - public void createTemporaryView(String path, Table view) { - tableEnv.createTemporaryView(path, view); - } - - @Override - public Table from(String path) { - return tableEnv.from(path); - } - - @Override - public String[] listCatalogs() { - return tableEnv.listCatalogs(); - } - - @Override - public String[] listModules() { - return tableEnv.listModules(); - } - - @Override - public String[] listDatabases() { - return tableEnv.listDatabases(); - } - - @Override - public String[] listTables() { - return tableEnv.listTables(); - } - - @Override - public String[] listViews() { - return tableEnv.listViews(); - } - - @Override - public String[] listTemporaryTables() { - return tableEnv.listTemporaryTables(); - } - - @Override - public String[] listTemporaryViews() { - return tableEnv.listTemporaryViews(); - } - - @Override - public String[] listUserDefinedFunctions() { - return tableEnv.listUserDefinedFunctions(); - } - - @Override - public String[] listFunctions() { - return tableEnv.listFunctions(); - } - - @Override - public boolean dropTemporaryTable(String path) { - return tableEnv.dropTemporaryTable(path); - } - - @Override - public boolean dropTemporaryView(String path) { - return tableEnv.dropTemporaryView(path); - } - - @Override - public String explainSql(String statement, ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, extraDetails); - } - - @Override - public Table sqlQuery(String query) { - return tableEnv.sqlQuery(query); - } - - @Override - public TableResult executeSql(String statement) { - return tableEnv.executeSql(statement); - } - - @Override - public String getCurrentCatalog() { - return tableEnv.getCurrentCatalog(); - } - - @Override - public void useCatalog(String catalogName) { - tableEnv.useCatalog(catalogName); - } - - @Override - public String getCurrentDatabase() { - return tableEnv.getCurrentDatabase(); - } - - @Override - public void useDatabase(String databaseName) { - tableEnv.useDatabase(databaseName); - } - - @Override - public TableConfig getConfig() { - return tableEnv.getConfig(); - } - - @Override - public String[] getCompletionHints(String statement, int position) { - return tableEnv.getCompletionHints(statement, position); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public Table scan(String... tablePath) { - return tableEnv.scan(tablePath); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void registerTable(String name, Table table) { - tableEnv.registerTable(name, table); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void registerFunction(String name, ScalarFunction function) { - tableEnv.registerFunction(name, function); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializerV2.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializerV2.java deleted file mode 100644 index f412ce4af8..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializerV2.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.FileUtils; -import org.apache.streampark.common.util.HdfsUtils; -import org.apache.streampark.common.util.PropertiesUtils; -import org.apache.streampark.flink.core.conf.FlinkConfiguration; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.util.ParameterTool; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -/** Initializes Flink streaming execution environment from application arguments. */ -public class FlinkStreamingInitializerV2 { - - final String[] args; - - StreamEnvConfigFunction javaStreamEnvConfFunc; - - private FlinkConfiguration configuration; - - private StreamExecutionEnvironment streamEnv; - - FlinkStreamingInitializerV2(String[] args) { - this.args = args; - } - - public static StreamingInitResult initialize(String[] args, StreamEnvConfigFunction config) { - FlinkStreamingInitializerV2 flinkInitializer = new FlinkStreamingInitializerV2(args); - flinkInitializer.javaStreamEnvConfFunc = config; - return new StreamingInitResult( - flinkInitializer.getConfiguration().parameter, flinkInitializer.getStreamEnv()); - } - - public static StreamingInitResult initialize(StreamEnvConfig args) { - FlinkStreamingInitializerV2 flinkInitializer = new FlinkStreamingInitializerV2(args.args); - flinkInitializer.javaStreamEnvConfFunc = args.conf; - return new StreamingInitResult( - flinkInitializer.getConfiguration().parameter, flinkInitializer.getStreamEnv()); - } - - ParameterTool getParameter() { - return getConfiguration().parameter; - } - - FlinkConfiguration getConfiguration() { - if (configuration == null) { - configuration = initParameter(); - } - return configuration; - } - - StreamExecutionEnvironment getStreamEnv() { - if (streamEnv == null) { - streamEnv = - StreamExecutionEnvironment.getExecutionEnvironment( - getConfiguration().envConfig); - if (javaStreamEnvConfFunc != null) { - javaStreamEnvConfFunc.configuration(streamEnv, getConfiguration().parameter); - } - streamEnv.getConfig().setGlobalJobParameters(getConfiguration().parameter); - } - return streamEnv; - } - - FlinkConfiguration initParameter() { - ParameterTool argsMap = ParameterTool.fromArgs(args); - String configFile = argsMap.get(ConfigKeys.KEY_APP_CONF(), null); - if (configFile == null || configFile.isEmpty()) { - throw new ExceptionInInitializerError( - "[StreamPark] Usage:can't find config,please set \"--conf $path \" in main arguments"); - } - Map configMap = parseConfig(configFile); - Map properConf = - extractConfigByPrefix(configMap, ConfigKeys.KEY_FLINK_PROPERTY_PREFIX()); - Map appConf = - extractConfigByPrefix(configMap, ConfigKeys.KEY_APP_PREFIX()); - - ParameterTool parameter = - ParameterTool.fromSystemProperties() - .mergeWith(ParameterTool.fromMap(properConf)) - .mergeWith(ParameterTool.fromMap(appConf)) - .mergeWith(argsMap); - - Configuration envConfig = Configuration.fromMap(properConf); - return new FlinkConfiguration(parameter, envConfig, null); - } - - Map parseConfig(String config) { - Map map; - if (config.startsWith("yaml://")) { - map = PropertiesUtils.fromYamlText(DeflaterUtils.unzipString(config.substring(7))); - } else if (config.startsWith("conf://")) { - map = PropertiesUtils.fromHoconText(DeflaterUtils.unzipString(config.substring(7))); - } else if (config.startsWith("prop://")) { - map = - PropertiesUtils.fromPropertiesText( - DeflaterUtils.unzipString(config.substring(7))); - } else if (config.startsWith("hdfs://")) { - try { - String text = HdfsUtils.read(config); - map = readConfig(config, text); - } catch (IOException e) { - throw new IllegalArgumentException( - "[StreamPark] Failed to read application config from HDFS: " + config, e); - } - } else { - File file = new File(config); - if (!file.exists()) { - throw new IllegalArgumentException( - "[StreamPark] Usage: application config file: " - + file - + " is not found!!!"); - } - try { - map = readConfig(config, FileUtils.readFile(file)); - } catch (IOException e) { - throw new IllegalArgumentException( - "[StreamPark] Failed to read application config file: " + config, e); - } - } - Map filtered = new HashMap<>(); - map.forEach( - (key, value) -> { - if (value != null && !value.isEmpty()) { - filtered.put(key, value); - } - }); - return filtered; - } - - private Map readConfig(String config, String text) { - String format = config.substring(config.lastIndexOf('.') + 1).toLowerCase(); - switch (format) { - case "yml": - case "yaml": - return PropertiesUtils.fromYamlText(text); - case "conf": - return PropertiesUtils.fromHoconText(text); - case "properties": - return PropertiesUtils.fromPropertiesText(text); - default: - throw new IllegalArgumentException( - "[StreamPark] Usage: application config file error,must be [yaml|conf|properties]"); - } - } - - Map extractConfigByPrefix(Map configMap, String prefix) { - Map map = new HashMap<>(); - configMap.forEach( - (key, value) -> { - if (key.startsWith(prefix)) { - map.put(key.substring(prefix.length()), value); - } - }); - return map; - } - - /** Streaming initialization result. */ - public static final class StreamingInitResult { - - public final ParameterTool parameter; - public final StreamExecutionEnvironment streamEnv; - - StreamingInitResult(ParameterTool parameter, StreamExecutionEnvironment streamEnv) { - this.parameter = parameter; - this.streamEnv = streamEnv; - } - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializerV2.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializerV2.java deleted file mode 100644 index 7799c1e2b1..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializerV2.java +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.enums.PlannerType; -import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.PropertiesUtils; -import org.apache.streampark.common.util.StreamParkLoggerFactory; -import org.apache.streampark.flink.core.conf.FlinkConfiguration; - -import org.apache.streampark.shaded.org.slf4j.Logger; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.TableConfig; -import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.util.ParameterTool; - -import java.io.File; -import java.lang.reflect.Method; -import java.util.HashMap; -import java.util.Map; - -/** Initializes Flink table and stream-table environments from application arguments. */ -public class FlinkTableInitializerV2 extends FlinkStreamingInitializerV2 { - - private static final Logger LOG = - StreamParkLoggerFactory.loggerFactory() - .getLogger(FlinkTableInitializerV2.class.getName()); - - private TableEnvConfigFunction javaTableEnvConfFunc; - - private EnvironmentSettings.Builder envSettingsBuilder; - - private TableEnvironment tableEnv; - - private StreamTableEnvironment streamTableEnv; - - FlinkTableInitializerV2(String[] args) { - super(args); - } - - public static TableInitResult initialize(TableEnvConfig args) { - FlinkTableInitializerV2 flinkInitializer = new FlinkTableInitializerV2(args.args); - flinkInitializer.javaTableEnvConfFunc = args.conf; - return new TableInitResult( - flinkInitializer.getConfiguration().parameter, flinkInitializer.getTableEnv()); - } - - public static StreamTableInitResult initialize(StreamTableEnvConfig args) { - FlinkTableInitializerV2 flinkInitializer = new FlinkTableInitializerV2(args.args); - flinkInitializer.javaStreamEnvConfFunc = args.streamConfig; - flinkInitializer.javaTableEnvConfFunc = args.tableConfig; - return new StreamTableInitResult( - flinkInitializer.getConfiguration().parameter, - flinkInitializer.getStreamEnv(), - flinkInitializer.getStreamTableEnv()); - } - - public static StreamTableInitResult initialize( - String[] args, - StreamEnvConfigFunction streamConfig, - TableEnvConfigFunction tableConfig) { - FlinkTableInitializerV2 flinkInitializer = new FlinkTableInitializerV2(args); - flinkInitializer.javaStreamEnvConfFunc = streamConfig; - flinkInitializer.javaTableEnvConfFunc = tableConfig; - return new StreamTableInitResult( - flinkInitializer.getConfiguration().parameter, - flinkInitializer.getStreamEnv(), - flinkInitializer.getStreamTableEnv()); - } - - TableEnvironment getTableEnv() { - if (tableEnv == null) { - LOG.info("job working in batch mode"); - EnvironmentSettings.Builder builder = getEnvSettingsBuilder(); - builder.inBatchMode(); - tableEnv = - FlinkParameterUtils.setAppName( - TableEnvironment.create(builder.build()), getParameter()); - applyTableEnvConfig(tableEnv.getConfig()); - } - return tableEnv; - } - - StreamTableEnvironment getStreamTableEnv() { - if (streamTableEnv == null) { - LOG.info("components should work in streaming mode"); - EnvironmentSettings.Builder builder = getEnvSettingsBuilder(); - builder.inStreamingMode(); - EnvironmentSettings setting = builder.build(); - - if (javaStreamEnvConfFunc != null) { - javaStreamEnvConfFunc.configuration(getStreamEnv(), getParameter()); - } - streamTableEnv = - FlinkParameterUtils.setAppName( - StreamTableEnvironment.create(getStreamEnv(), setting), getParameter()); - applyTableEnvConfig(streamTableEnv.getConfig()); - } - return streamTableEnv; - } - - private void applyTableEnvConfig(TableConfig config) { - if (javaTableEnvConfFunc != null) { - javaTableEnvConfFunc.configuration(config, getParameter()); - } - } - - private EnvironmentSettings.Builder getEnvSettingsBuilder() { - if (envSettingsBuilder == null) { - envSettingsBuilder = buildEnvSettings(getParameter()); - } - return envSettingsBuilder; - } - - private EnvironmentSettings.Builder buildEnvSettings(ParameterTool parameter) { - EnvironmentSettings.Builder builder = EnvironmentSettings.newInstance(); - - PlannerType plannerType = PlannerType.BLINK; - String plannerName = parameter.get(ConfigKeys.KEY_FLINK_TABLE_PLANNER(), null); - if (plannerName != null && !plannerName.isEmpty()) { - try { - plannerType = PlannerType.withName(plannerName); - } catch (IllegalArgumentException e) { - plannerType = PlannerType.BLINK; - } - } - - switch (plannerType) { - case BLINK: - invokePlannerMethod(builder, "useBlinkPlanner", "blinkPlanner will be used."); - break; - case OLD: - invokePlannerMethod(builder, "useOldPlanner", "useOldPlanner will be used."); - break; - case ANY: - invokePlannerMethod(builder, "useAnyPlanner", "useAnyPlanner will be used."); - break; - default: - break; - } - - String flinkConf = parameter.get(ConfigKeys.KEY_FLINK_CONF(), null); - if (flinkConf == null || flinkConf.isEmpty()) { - throw new ExceptionInInitializerError( - "[StreamPark] Usage:can't find config,please set \"--flink.conf $conf \" in main arguments"); - } - builder.withConfiguration( - Configuration.fromMap( - PropertiesUtils.fromYamlText(DeflaterUtils.unzipString(flinkConf)))); - - String catalog = parameter.get(ConfigKeys.KEY_FLINK_TABLE_CATALOG(), null); - String database = parameter.get(ConfigKeys.KEY_FLINK_TABLE_DATABASE(), null); - if (catalog != null && database != null) { - LOG.info("with built in catalog: {}", catalog); - LOG.info("with built in database: {}", database); - builder.withBuiltInCatalogName(catalog); - builder.withBuiltInDatabaseName(database); - } else if (catalog != null) { - LOG.info("with built in catalog: {}", catalog); - builder.withBuiltInCatalogName(catalog); - } else if (database != null) { - LOG.info("with built in database: {}", database); - builder.withBuiltInDatabaseName(database); - } - return builder; - } - - private void invokePlannerMethod( - EnvironmentSettings.Builder builder, String methodName, String successMessage) { - try { - Method method = builder.getClass().getDeclaredMethod(methodName); - method.setAccessible(true); - method.invoke(builder); - if (successMessage != null) { - LOG.info(successMessage); - } - } catch (NoSuchMethodException e) { - LOG.warn("{} deprecated", methodName); - } catch (ReflectiveOperationException e) { - LOG.warn("Failed to invoke {} on EnvironmentSettings.Builder", methodName, e); - } - } - - @Override - FlinkConfiguration initParameter() { - ParameterTool argsMap = ParameterTool.fromArgs(args); - String configFile = argsMap.get(ConfigKeys.KEY_APP_CONF(), null); - FlinkConfiguration configuration; - if (configFile == null || configFile.isEmpty()) { - LOG.warn("Usage:can't find config,you can set \"--conf $path \" in main arguments"); - ParameterTool parameter = ParameterTool.fromSystemProperties().mergeWith(argsMap); - configuration = - new FlinkConfiguration(parameter, new Configuration(), new Configuration()); - } else { - Map configMap = parseConfig(configFile); - Map sqlConf = new HashMap<>(); - configMap.forEach( - (key, value) -> { - if (key.startsWith(ConfigKeys.KEY_SQL_PREFIX())) { - sqlConf.put(key.substring(ConfigKeys.KEY_SQL_PREFIX().length()), value); - } - }); - - Map properConf = - extractConfigByPrefix(configMap, ConfigKeys.KEY_FLINK_PROPERTY_PREFIX()); - Map appConf = - extractConfigByPrefix(configMap, ConfigKeys.KEY_APP_PREFIX()); - Map tableConf = - extractConfigByPrefix(configMap, ConfigKeys.KEY_FLINK_TABLE_PREFIX()); - - Configuration tableConfig = Configuration.fromMap(tableConf); - Configuration envConfig = Configuration.fromMap(properConf); - - ParameterTool parameter = - ParameterTool.fromSystemProperties() - .mergeWith(ParameterTool.fromMap(properConf)) - .mergeWith(ParameterTool.fromMap(tableConf)) - .mergeWith(ParameterTool.fromMap(appConf)) - .mergeWith(ParameterTool.fromMap(sqlConf)) - .mergeWith(argsMap); - - configuration = new FlinkConfiguration(parameter, envConfig, tableConfig); - } - - String flinkSql = configuration.parameter.get(ConfigKeys.KEY_FLINK_SQL(), null); - if (flinkSql == null) { - return configuration; - } - - try { - String value = DeflaterUtils.unzipString(flinkSql); - return configuration.withParameter( - configuration.parameter.mergeWith( - ParameterTool.fromMap( - Map.of(ConfigKeys.KEY_FLINK_SQL(), value)))); - } catch (Exception ignored) { - File sqlFile = new File(flinkSql); - try { - Map value = - PropertiesUtils.fromYamlFile(sqlFile.getAbsolutePath()); - return configuration.withParameter( - configuration.parameter.mergeWith(ParameterTool.fromMap(value))); - } catch (Exception e) { - throw new IllegalArgumentException("[StreamPark] init sql error." + e, e); - } - } - } - - /** Table initialization result. */ - public static final class TableInitResult { - - public final ParameterTool parameter; - public final TableEnvironment tableEnv; - - public TableInitResult(ParameterTool parameter, TableEnvironment tableEnv) { - this.parameter = parameter; - this.tableEnv = tableEnv; - } - } - - /** Stream-table initialization result. */ - public static final class StreamTableInitResult { - - public final ParameterTool parameter; - public final StreamExecutionEnvironment streamEnv; - public final StreamTableEnvironment streamTableEnv; - - public StreamTableInitResult( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment streamTableEnv) { - this.parameter = parameter; - this.streamEnv = streamEnv; - this.streamTableEnv = streamTableEnv; - } - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/SqlCommand.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/SqlCommand.java deleted file mode 100644 index ddc5f5abf4..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/SqlCommand.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.commons.lang3.StringUtils; - -import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** Flink SQL command types. */ -public enum SqlCommand { - - // ---- SELECT Statements ----------------------------------------------------------------- - SELECT("select", "(SELECT\\s+.+)"), - - // ---- CREATE Statements ----------------------------------------------------------------- - CREATE_TABLE("create table", "(CREATE\\s+(TEMPORARY\\s+|)TABLE\\s+.+)"), - CREATE_CATALOG("create catalog", "(CREATE\\s+CATALOG\\s+.+)"), - CREATE_DATABASE("create database", "(CREATE\\s+DATABASE\\s+.+)"), - CREATE_VIEW( - "create view", - "(CREATE\\s+(TEMPORARY\\s+|)VIEW\\s+(IF\\s+NOT\\s+EXISTS\\s+|)(\\S+)\\s+AS\\s+SELECT\\s+.+)"), - CREATE_FUNCTION( - "create function", - "(CREATE\\s+(TEMPORARY\\s+|TEMPORARY\\s+SYSTEM\\s+|)FUNCTION\\s+(IF\\s+NOT\\s+EXISTS\\s+|)(\\S+)\\s+AS\\s+.*)"), - - // ---- DROP Statements ------------------------------------------------------------------- - DROP_CATALOG("drop catalog", "(DROP\\s+CATALOG\\s+.+)"), - DROP_TABLE("drop table", "(DROP\\s+(TEMPORARY\\s+|)TABLE\\s+.+)"), - DROP_DATABASE("drop database", "(DROP\\s+DATABASE\\s+.+)"), - DROP_VIEW("drop view", "(DROP\\s+(TEMPORARY\\s+|)VIEW\\s+.+)"), - DROP_FUNCTION( - "drop function", "(DROP\\s+(TEMPORARY\\s+|TEMPORARY\\s+SYSTEM\\s+|)FUNCTION\\s+.+)"), - - // ---- ALTER Statements ------------------------------------------------------------------ - ALTER_TABLE("alter table", "(ALTER\\s+TABLE\\s+.+)"), - ALTER_VIEW("alter view", "(ALTER\\s+VIEW\\s+.+)"), - ALTER_DATABASE("alter database", "(ALTER\\s+DATABASE\\s+.+)"), - ALTER_FUNCTION( - "alter function", - "(ALTER\\s+(TEMPORARY\\s+|TEMPORARY\\s+SYSTEM\\s+|)FUNCTION\\s+.+)"), - - // ---- INSERT Statement ------------------------------------------------------------------ - INSERT("insert", "(INSERT\\s+(INTO|OVERWRITE)\\s+.+)"), - - // ---- DESCRIBE Statement ---------------------------------------------------------------- - DESC("desc", "(DESC\\s+.+)"), - DESCRIBE("describe", "(DESCRIBE\\s+.+)"), - - // ---- EXPLAIN Statement ----------------------------------------------------------------- - EXPLAIN("explain", "(EXPLAIN\\s+.+)"), - - // ---- USE Statements -------------------------------------------------------------------- - USE_CATALOG("use catalog", "(USE\\s+CATALOG\\s+.+)"), - USE_MODULES("use modules", "(USE\\s+MODULES\\s+.+)"), - USE_DATABASE("use database", "(USE\\s+(?!(CATALOG|MODULES)).+)"), - - // ---- SHOW Statements ------------------------------------------------------------------- - SHOW_CATALOGS("show catalogs", "(SHOW\\s+CATALOGS\\s*)"), - SHOW_CURRENT_CATALOG("show current catalog", "(SHOW\\s+CURRENT\\s+CATALOG\\s*)"), - SHOW_DATABASES("show databases", "(SHOW\\s+DATABASES\\s*)"), - SHOW_CURRENT_DATABASE("show current database", "(SHOW\\s+CURRENT\\s+DATABASE\\s*)"), - SHOW_TABLES("show tables", "(SHOW\\s+TABLES.*)"), - SHOW_CREATE_TABLE("show create table", "(SHOW\\s+CREATE\\s+TABLE\\s+.+)"), - SHOW_COLUMNS("show columns", "(SHOW\\s+COLUMNS\\s+.+)"), - SHOW_VIEWS("show views", "(SHOW\\s+VIEWS\\s*)"), - SHOW_CREATE_VIEW("show create view", "(SHOW\\s+CREATE\\s+VIEW\\s+.+)"), - SHOW_FUNCTIONS("show functions", "(SHOW\\s+(USER\\s+|)FUNCTIONS\\s*)"), - SHOW_MODULES("show modules", "(SHOW\\s+(FULL\\s+|)MODULES\\s*)"), - - // ---- LOAD Statements ------------------------------------------------------------------- - LOAD_MODULE("load module", "(LOAD\\s+MODULE\\s+.+)"), - - // ---- UNLOAD Statements ----------------------------------------------------------------- - UNLOAD_MODULE("unload module", "(UNLOAD\\s+MODULE\\s+.+)"), - - // ---- SET Statements -------------------------------------------------------------------- - SET( - "set", - "SET(\\s+(\\S+)\\s*=(.*))?", - groups -> { - if (groups.length < 3) { - return Optional.empty(); - } - if (groups[0] == null) { - return Optional.of(new String[]{cleanUp(groups[0])}); - } - return Optional.of(new String[]{cleanUp(groups[1]), cleanUp(groups[2])}); - }), - - // ---- RESET Statements ------------------------------------------------------------------ - RESET("reset", "RESET\\s+'(.*)'"), - RESET_ALL("reset all", "RESET", groups -> Optional.of(new String[]{"ALL"})), - - // ---- INSERT SET Statements ------------------------------------------------------------- - /** @deprecated SQL Client syntax; not supported on this platform. */ - @Deprecated(since = "2.1.0", forRemoval = false) - BEGIN_STATEMENT_SET( - "begin statement set", "BEGIN\\s+STATEMENT\\s+SET", SqlCommandConverters.NO_OPERANDS), - /** @deprecated SQL Client syntax; not supported on this platform. */ - @Deprecated(since = "2.1.0", forRemoval = false) - END_STATEMENT_SET("end statement set", "END", SqlCommandConverters.NO_OPERANDS), - - // Since: 2.1.2 for flink 1.18 - DELETE("delete", "(DELETE\\s+FROM\\s+.+)"), - UPDATE("update", "(UPDATE\\s+.+)"); - - private static final int PATTERN_FLAGS = Pattern.CASE_INSENSITIVE | Pattern.DOTALL; - - private final String name; - private final String regex; - private final SqlCommandConverter converter; - private Matcher matcher; - - SqlCommand(String name, String regex) { - this(name, regex, SqlCommandConverters.DEFAULT); - } - - SqlCommand(String name, String regex, SqlCommandConverter converter) { - this.name = name; - this.regex = regex; - this.converter = converter; - } - - /** Command label (e.g. {@code "select"}, {@code "create table"}). */ - public String getName() { - return name; - } - - public String getRegex() { - return regex; - } - - public SqlCommandConverter getConverter() { - return converter; - } - - public Matcher getMatcher() { - return matcher; - } - - public boolean matches(String input) { - if (StringUtils.isBlank(regex)) { - return false; - } - Pattern pattern = Pattern.compile(regex, PATTERN_FLAGS); - matcher = pattern.matcher(input); - return matcher.matches(); - } - - /** Resolve the first matching command for the given statement. */ - public static SqlCommand get(String stmt) { - for (SqlCommand command : values()) { - if (command.matches(stmt)) { - return command; - } - } - return null; - } - - static String cleanUp(String sql) { - return sql.trim().replaceAll("^(['\"])|(['\"])$", ""); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/TableEnvConfigFunction.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/TableEnvConfigFunction.java deleted file mode 100644 index 0e57b74b5c..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/TableEnvConfigFunction.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.table.api.TableConfig; -import org.apache.flink.util.ParameterTool; - -@FunctionalInterface -public interface TableEnvConfigFunction { - - /** - * When used to initialize the TableEnvironment, it can be used to implement this function and - * customize the parameters to be set... - * - * @param tableConfig - * @param parameterTool - */ - void configuration(TableConfig tableConfig, ParameterTool parameterTool); -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/conf/FlinkConfiguration.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/conf/FlinkConfiguration.java deleted file mode 100644 index c55457631b..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/conf/FlinkConfiguration.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core.conf; - -import org.apache.flink.configuration.Configuration; -import org.apache.flink.util.ParameterTool; - -/** Flink runtime configuration holder. */ -public class FlinkConfiguration { - - public final ParameterTool parameter; - public final Configuration envConfig; - public final Configuration tableConfig; - - public FlinkConfiguration( - ParameterTool parameter, Configuration envConfig, Configuration tableConfig) { - this.parameter = parameter; - this.envConfig = envConfig; - this.tableConfig = tableConfig; - } - - public FlinkConfiguration withParameter(ParameterTool parameter) { - return new FlinkConfiguration(parameter, envConfig, tableConfig); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/util/FlinkUtils.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/util/FlinkUtils.java deleted file mode 100644 index 441dbf3d7b..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/util/FlinkUtils.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.util; - -import org.apache.flink.api.common.state.ListState; -import org.apache.flink.api.common.state.ListStateDescriptor; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.configuration.CheckpointingOptions; -import org.apache.flink.runtime.state.FunctionInitializationContext; -import org.apache.flink.util.TimeUtils; - -import java.io.File; -import java.time.Duration; -import java.util.Arrays; -import java.util.Map; -import java.util.stream.Collectors; - -/** Flink utility methods. */ -public final class FlinkUtils { - - private FlinkUtils() { - } - - public static ListState getUnionListState( - FunctionInitializationContext context, - String descriptorName, - TypeInformation typeInformation) { - try { - return context.getOperatorStateStore() - .getUnionListState( - new ListStateDescriptor<>( - descriptorName, typeInformation.getTypeClass())); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public static String getFlinkDistJar(String flinkHome) { - String[] jars = - new File(flinkHome + "/lib") - .list((dir, name) -> name.matches("flink-dist.*\\.jar")); - if (jars == null || jars.length == 0) { - throw new IllegalArgumentException( - "[StreamPark] can no found flink-dist jar in " + flinkHome + "/lib"); - } - if (jars.length == 1) { - return flinkHome + "/lib/" + jars[0]; - } - throw new IllegalArgumentException( - "[StreamPark] found multiple flink-dist jar in " - + flinkHome - + "/lib,[" - + Arrays.stream(jars).collect(Collectors.joining(",")) - + "]"); - } - - public static boolean isCheckpointEnabled(Map map) { - Duration checkpointInterval = - TimeUtils.parseDuration( - map.getOrDefault( - CheckpointingOptions.CHECKPOINTING_INTERVAL.key(), "0ms")); - return checkpointInterval.toMillis() > 0; - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/test/java/org/apache/streampark/RegExpTest.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/test/java/org/apache/streampark/RegExpTest.java deleted file mode 100644 index dc7e648e1d..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/test/java/org/apache/streampark/RegExpTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark; - -import org.junit.jupiter.api.Test; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** some simple tests */ -class RegExpTest { - - /** - * Case insensitive, matches everything as one line, . matches newlines, note: \s matches any - * whitespace character, including newlines - */ - public static final int DEFAULT_PATTERN_FLAGS = Pattern.CASE_INSENSITIVE | Pattern.DOTALL; - - /** - * CREATE CATALOG catalog_name WITH (key1=val1, key2=val2, ...)
- * Example：create catalog hive_catalog with('name' = 'my_hive', 'conf' = '/home/hive/conf') - */ - private static final Pattern CREATE_HIVE_CATALOG = - Pattern.compile("CREATE\\s+CATALOG\\s+\\S[\\s\\S]*", DEFAULT_PATTERN_FLAGS); - - @Test - void testCreateHiveCatalog() { - String str = "create catalog hive with (\n" - + " 'type' = 'hive',\n" - + " 'hadoop-conf-dir' = 'D:\\IDEAWorkspace\\work\\baishan\\log\\data-max\\src\\main\\resources',\n" - + " 'hive-conf-dir' = 'D:\\IDEAWorkspace\\work\\baishan\\log\\data-max\\src\\main\\resources'\n" - + ")"; - Matcher matcher = CREATE_HIVE_CATALOG.matcher(str); - System.out.println(matcher.matches()); - } - - /** - * CREATE [TEMPORARY|TEMPORARY SYSTEM] FUNCTION [IF NOT EXISTS] - * [catalog_name.][db_name.]function_name AS identifier [LANGUAGE JAVA|SCALA|PYTHON]
- * Example：create function test_fun as com.flink.testFun - */ - private static final Pattern CREATE_FUNCTION = - Pattern.compile( - "CREATE\\s+(?:TEMPORARY\\s+(?:SYSTEM\\s+)?)?FUNCTION\\s+(?:IF NOT EXISTS\\s+)?" - + "([A-Za-z][A-Za-z\\d.\\-_]*)\\s+AS\\s+'([^']+)'\\s+LANGUAGE\\s+(JAVA|SCALA|PYTHON)", - DEFAULT_PATTERN_FLAGS); - - @Test - void testCreateFunction() { - String str = - "create function if not exists hive.get_json_value as 'com.flink.function.JsonValueFunction' language java"; - Matcher matcher = CREATE_FUNCTION.matcher(str); - System.out.println(matcher.matches()); - } - - /** USE [catalog_name.]database_name */ - private static final Pattern USE_DATABASE = - Pattern.compile("USE\\s+(?!(?:CATALOG|MODULES)\\b)\\S[\\s\\S]*", DEFAULT_PATTERN_FLAGS); - - @Test - void testUseDatabase() { - String str = "use modul.a "; - Matcher matcher = USE_DATABASE.matcher(str); - System.out.println(matcher.matches()); - } - - /** SHOW [USER] FUNCTIONS */ - private static final Pattern SHOW_FUNCTIONS = - Pattern.compile("SHOW\\s+(?:USER\\s+)?FUNCTIONS\\b", DEFAULT_PATTERN_FLAGS); - - @Test - void testShowFunction() { - String str = "show user functions"; - Matcher matcher = SHOW_FUNCTIONS.matcher(str); - System.out.println(matcher.matches()); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/pom.xml index c7734c6d8e..81742c6755 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/pom.xml +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/pom.xml @@ -28,6 +28,10 @@ streampark-flink-shims-base StreamPark : Flink Shims Base + + 1.18.0 + + diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/FlinkJobParameters.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/FlinkJobParameters.java new file mode 100644 index 0000000000..4eb91ba99c --- /dev/null +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/FlinkJobParameters.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.configuration; + +import org.apache.streampark.common.configuration.ConfigException; +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.Configuration; +import org.apache.streampark.common.configuration.ReadableConfig; + +import org.apache.flink.api.common.ExecutionConfig; + +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +/** + * Thin Flink adapter over StreamPark's immutable application configuration. + * + *

Flink requires global job parameters to extend {@link + * ExecutionConfig.GlobalJobParameters}. All parsing, precedence, conversion, and validation remain + * in the common configuration subsystem; this class only bridges that snapshot into Flink. + */ +public final class FlinkJobParameters extends ExecutionConfig.GlobalJobParameters + implements + ReadableConfig { + + private static final long serialVersionUID = 1L; + + private final Configuration configuration; + + private FlinkJobParameters(Configuration configuration) { + this.configuration = + Objects.requireNonNull(configuration, "configuration must not be null"); + } + + /** + * Creates a Flink parameter adapter for an immutable application snapshot. + * + * @param configuration application configuration snapshot + * @return Flink global-job-parameter adapter + */ + public static FlinkJobParameters of(Configuration configuration) { + return new FlinkJobParameters(configuration); + } + + /** + * Returns the underlying immutable application configuration. + * + * @return application configuration snapshot + */ + public Configuration configuration() { + return configuration; + } + + @Override + public T get(ConfigOption option) { + return configuration.get(option); + } + + @Override + public Optional getOptional(ConfigOption option) { + return configuration.getOptional(option); + } + + @Override + public boolean contains(ConfigOption option) { + return configuration.contains(option); + } + + /** + * Returns a required parameter by its exact key. + * + *

This compatibility accessor does not apply option defaults or fallback keys. New code + * should prefer {@link #get(ConfigOption)} when typed metadata is available. + * + * @param key exact application parameter key + * @return string representation of the configured value + * @throws ConfigException when the key is absent + */ + public String get(String key) { + return configuration.getOptionalString(key) + .orElseThrow(() -> new ConfigException("Required application parameter '" + key + "' is missing")); + } + + /** + * Returns a parameter by its exact key or a caller-supplied default value. + * + * @param key exact application parameter key + * @param defaultValue value returned when the key is absent + * @return configured or default value + */ + public String get(String key, String defaultValue) { + return configuration.getOptionalString(key).orElse(defaultValue); + } + + /** + * Returns whether an exact parameter key is present. + * + * @param key exact application parameter key + * @return whether the key is present + */ + public boolean has(String key) { + return configuration.containsKey(key); + } + + @Override + public Map toMap() { + return configuration.toMap(); + } + + @Override + public String toString() { + return configuration.toString(); + } +} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/FlinkRuntimeConfiguration.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/FlinkRuntimeConfiguration.java new file mode 100644 index 0000000000..3e3099febb --- /dev/null +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/FlinkRuntimeConfiguration.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.configuration; + +import org.apache.streampark.common.configuration.Configuration; + +import java.util.Objects; + +/** + * Complete, immutable configuration assembled for one Flink application initialization. + * + *

The StreamPark application snapshot remains separate from Flink's native environment and + * table configurations so namespace extraction cannot silently overwrite application arguments. + */ +public final class FlinkRuntimeConfiguration { + + private final Configuration applicationConfiguration; + private final FlinkJobParameters jobParameters; + private final org.apache.flink.configuration.Configuration environmentConfiguration; + private final org.apache.flink.configuration.Configuration tableConfiguration; + + /** + * Creates the configuration views for one initialized application. + * + * @param applicationConfiguration StreamPark application parameters + * @param environmentConfiguration native Flink execution configuration + * @param tableConfiguration native Flink table configuration, or {@code null} for DataStream + * applications + */ + public FlinkRuntimeConfiguration( + Configuration applicationConfiguration, + org.apache.flink.configuration.Configuration environmentConfiguration, + org.apache.flink.configuration.Configuration tableConfiguration) { + this.applicationConfiguration = + Objects.requireNonNull( + applicationConfiguration, "applicationConfiguration must not be null"); + this.jobParameters = FlinkJobParameters.of(applicationConfiguration); + this.environmentConfiguration = + Objects.requireNonNull( + environmentConfiguration, "environmentConfiguration must not be null"); + this.tableConfiguration = tableConfiguration; + } + + /** + * Returns StreamPark application parameters before Flink adaptation. + * + * @return immutable application configuration + */ + public Configuration applicationConfiguration() { + return applicationConfiguration; + } + + /** + * Returns application parameters exposed through Flink's global job parameters. + * + * @return Flink parameter adapter + */ + public FlinkJobParameters jobParameters() { + return jobParameters; + } + + /** + * Returns the native Flink execution configuration. + * + * @return native execution configuration + */ + public org.apache.flink.configuration.Configuration environmentConfiguration() { + return environmentConfiguration; + } + + /** + * Returns the native table configuration. + * + * @return native table configuration, or {@code null} for DataStream applications + */ + public org.apache.flink.configuration.Configuration tableConfiguration() { + return tableConfiguration; + } + + /** + * Returns a copy with different application parameters and the same native Flink views. + * + * @param configuration replacement application configuration + * @return new runtime configuration + */ + public FlinkRuntimeConfiguration withApplicationConfiguration(Configuration configuration) { + return new FlinkRuntimeConfiguration( + configuration, environmentConfiguration, tableConfiguration); + } +} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/package-info.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/package-info.java new file mode 100644 index 0000000000..a25cd5fa04 --- /dev/null +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/package-info.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Flink-specific configuration declarations, runtime views, and submission adapters. + * + *

StreamPark application options and native Flink configuration remain separate until the + * initializer creates a {@link org.apache.streampark.flink.configuration.FlinkRuntimeConfiguration} + * for a concrete job. {@link FlinkConfigurationUtils} + * normalizes Flink's legacy and standard YAML file formats before those views are assembled. + * + *

Installations before Flink 1.19 use flat {@code flink-conf.yaml}. Flink 1.19 and 1.20 prefer + * that legacy file when present and otherwise use standard {@code config.yaml}. Flink 2.0 and + * later require {@code config.yaml}. The selected format is carried with compressed configuration + * content because a runtime payload no longer retains its source filename. + */ +package org.apache.streampark.flink.configuration; + +import org.apache.streampark.common.util.FlinkConfigurationUtils; diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkClientTrait.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkClient.java similarity index 96% rename from streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkClientTrait.java rename to streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkClient.java index f7ae7357e9..21cde9be52 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkClientTrait.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkClient.java @@ -24,11 +24,11 @@ import java.util.concurrent.CompletableFuture; /** Flink cluster client savepoint operations. */ -public abstract class FlinkClientTrait { +public abstract class AbstractFlinkClient { protected final ClusterClient clusterClient; - protected FlinkClientTrait(ClusterClient clusterClient) { + protected AbstractFlinkClient(ClusterClient clusterClient) { this.clusterClient = clusterClient; } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClientTrait.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkKubernetesClient.java similarity index 92% rename from streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClientTrait.java rename to streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkKubernetesClient.java index 02cbd0a563..e27e33aa24 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClientTrait.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkKubernetesClient.java @@ -23,11 +23,11 @@ import java.util.Optional; /** Flink Kubernetes client operations. */ -public abstract class FlinkKubernetesClientTrait { +public abstract class AbstractFlinkKubernetesClient { protected final FlinkKubeClient kubeClient; - protected FlinkKubernetesClientTrait(FlinkKubeClient kubeClient) { + protected AbstractFlinkKubernetesClient(FlinkKubeClient kubeClient) { this.kubeClient = kubeClient; } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkStreamTable.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkStreamTable.java new file mode 100644 index 0000000000..99c0c4c950 --- /dev/null +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkStreamTable.java @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.core; + +import org.apache.streampark.common.util.Utils; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.util.FlinkParameterUtils; + +import org.apache.flink.api.common.JobExecutionResult; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.flink.streaming.api.datastream.DataStream; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.Schema; +import org.apache.flink.table.api.Table; +import org.apache.flink.table.api.bridge.java.StreamStatementSet; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.table.catalog.CatalogDescriptor; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.expressions.Expression; +import org.apache.flink.table.types.AbstractDataType; +import org.apache.flink.types.Row; + +import java.util.function.Consumer; + +/** + * Shared stream-table adapter for all supported Flink versions. + * + *

This class deliberately contains only methods whose contracts are compatible across every + * supported Flink version. APIs added, changed, or removed between Flink releases belong in the + * concrete version-specific adapters so the shared bytecode remains linkable with each Flink + * distribution. Once a Table has been converted to a DataStream, the DataStream job must be + * executed using the execute method of the StreamExecutionEnvironment. + */ +@SuppressWarnings("java:S100") +public abstract class AbstractFlinkStreamTable extends AbstractFlinkTable implements StreamTableEnvironment { + + private final StreamExecutionEnvironment streamEnv; + + private final StreamTableEnvironment tableEnv; + + /** Whether a table has been converted to a DataStream. */ + public boolean isConvertedToDataStream; + + protected AbstractFlinkStreamTable( + FlinkJobParameters parameter, + StreamExecutionEnvironment streamEnv, + StreamTableEnvironment tableEnv) { + super(parameter, tableEnv); + this.streamEnv = streamEnv; + this.tableEnv = tableEnv; + } + + protected StreamExecutionEnvironment getStreamEnv() { + return streamEnv; + } + + protected StreamTableEnvironment getStreamTableEnv() { + return tableEnv; + } + + @Override + public abstract StreamStatementSet createStatementSet(); + + /** Recommended API to start tasks. */ + public JobExecutionResult start() { + return start(null); + } + + public JobExecutionResult start(String name) { + String appName = FlinkParameterUtils.getAppName(parameter, name, true); + return execute(appName); + } + + /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ + @Deprecated(since = "2.1.0", forRemoval = false) + public JobExecutionResult execute(String jobName) { + Utils.printLogo("FlinkStreamTable " + jobName + " Starting..."); + if (isConvertedToDataStream) { + try { + return streamEnv.execute(jobName); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return null; + } + + /** Executes SQL configured by the default application SQL key. */ + public void sql() { + sql(null); + } + + /** Executes SQL configured by the supplied parameter key. */ + public void sql(String sqlKey) { + sql(sqlKey, null); + } + + /** Executes SQL configured by the supplied parameter key and reports metadata to a callback. */ + public void sql(String sqlKey, Consumer callback) { + FlinkSqlExecutor.executeSql(sqlKey, parameter, this, callback); + } + + @Override + public Table fromDataStream(DataStream dataStream) { + return getStreamTableEnv().fromDataStream(dataStream); + } + + @Override + public Table fromDataStream(DataStream dataStream, Schema schema) { + return getStreamTableEnv().fromDataStream(dataStream, schema); + } + + @Override + public Table fromChangelogStream(DataStream dataStream) { + return getStreamTableEnv().fromChangelogStream(dataStream); + } + + @Override + public Table fromChangelogStream(DataStream dataStream, Schema schema) { + return getStreamTableEnv().fromChangelogStream(dataStream, schema); + } + + @Override + public Table fromChangelogStream(DataStream dataStream, Schema schema, ChangelogMode changelogMode) { + return getStreamTableEnv().fromChangelogStream(dataStream, schema, changelogMode); + } + + @Override + public void createTemporaryView(String path, DataStream dataStream) { + getStreamTableEnv().createTemporaryView(path, dataStream); + } + + @Override + public void createTemporaryView(String path, DataStream dataStream, Schema schema) { + getStreamTableEnv().createTemporaryView(path, dataStream, schema); + } + + @Override + public DataStream toDataStream(Table table) { + isConvertedToDataStream = true; + return getStreamTableEnv().toDataStream(table); + } + + @Override + public DataStream toDataStream(Table table, Class targetClass) { + isConvertedToDataStream = true; + return getStreamTableEnv().toDataStream(table, targetClass); + } + + @Override + public DataStream toDataStream(Table table, AbstractDataType targetDataType) { + isConvertedToDataStream = true; + return getStreamTableEnv().toDataStream(table, targetDataType); + } + + @Override + public DataStream toChangelogStream(Table table) { + isConvertedToDataStream = true; + return getStreamTableEnv().toChangelogStream(table); + } + + @Override + public DataStream toChangelogStream(Table table, Schema targetSchema) { + isConvertedToDataStream = true; + return getStreamTableEnv().toChangelogStream(table, targetSchema); + } + + @Override + public DataStream toChangelogStream(Table table, Schema targetSchema, ChangelogMode changelogMode) { + isConvertedToDataStream = true; + return getStreamTableEnv().toChangelogStream(table, targetSchema, changelogMode); + } + + @Override + public Table fromDataStream(DataStream dataStream, Expression... fields) { + return getStreamTableEnv().fromDataStream(dataStream, fields); + } + + @Override + public void createTemporaryView(String path, DataStream dataStream, Expression... fields) { + getStreamTableEnv().createTemporaryView(path, dataStream, fields); + } + + @Override + public DataStream toAppendStream(Table table, Class clazz) { + isConvertedToDataStream = true; + return getStreamTableEnv().toAppendStream(table, clazz); + } + + @Override + public DataStream toAppendStream(Table table, TypeInformation typeInfo) { + isConvertedToDataStream = true; + return getStreamTableEnv().toAppendStream(table, typeInfo); + } + + @Override + public DataStream> toRetractStream(Table table, Class clazz) { + isConvertedToDataStream = true; + return getStreamTableEnv().toRetractStream(table, clazz); + } + + @Override + public DataStream> toRetractStream(Table table, TypeInformation typeInfo) { + isConvertedToDataStream = true; + return getStreamTableEnv().toRetractStream(table, typeInfo); + } + + @Override + public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { + getStreamTableEnv().createCatalog(catalogName, catalogDescriptor); + } +} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkTable.java similarity index 81% rename from streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java rename to streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkTable.java index e655b37347..7292e46635 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/AbstractFlinkTable.java @@ -18,6 +18,8 @@ package org.apache.streampark.flink.core; import org.apache.streampark.common.util.Utils; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.util.FlinkParameterUtils; import org.apache.flink.api.common.JobExecutionResult; import org.apache.flink.table.api.CompiledPlan; @@ -40,19 +42,20 @@ import org.apache.flink.table.module.ModuleEntry; import org.apache.flink.table.resource.ResourceUri; import org.apache.flink.table.types.AbstractDataType; -import org.apache.flink.util.ParameterTool; + +import javax.annotation.Nullable; import java.util.List; import java.util.Optional; -/** Base table environment trait with SQL execution helpers. */ -public abstract class FlinkTableTrait implements TableEnvironment { +/** Base table environment adapter with SQL execution helpers. */ +public abstract class AbstractFlinkTable implements TableEnvironment { - public final ParameterTool parameter; + public final FlinkJobParameters parameter; private final TableEnvironment tableEnv; - protected FlinkTableTrait(ParameterTool parameter, TableEnvironment tableEnv) { + protected AbstractFlinkTable(FlinkJobParameters parameter, TableEnvironment tableEnv) { this.parameter = parameter; this.tableEnv = tableEnv; } @@ -71,177 +74,172 @@ public JobExecutionResult execute(String jobName) { return null; } - public void sql(String sql) { - FlinkSqlExecutor.executeSql(sql, parameter, this); + /** Executes SQL configured by the default application SQL key. */ + public void sql() { + FlinkSqlExecutor.executeSql(parameter, this); + } + + /** Executes SQL configured by the supplied parameter key. */ + public void sql(String sqlKey) { + FlinkSqlExecutor.executeSql(sqlKey, parameter, this); } + // overview methods + @Override - public Table fromValues(Expression... values) { + public Table fromValues(Object... values) { return tableEnv.fromValues(values); } @Override - public Table fromValues(AbstractDataType rowType, Expression... values) { + public Table fromValues(AbstractDataType rowType, Object... values) { return tableEnv.fromValues(rowType, values); } @Override - public Table fromValues(Iterable values) { - return tableEnv.fromValues(values); + public String explainSql(String statement, ExplainDetail... extraDetails) { + return tableEnv.explainSql(statement, extraDetails); } @Override - public Table fromValues(AbstractDataType rowType, Iterable values) { - return tableEnv.fromValues(rowType, values); + public TableResult executePlan(PlanReference planReference) throws TableException { + return tableEnv.executePlan(planReference); } @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - tableEnv.createCatalog(catalogName, catalogDescriptor); + public Table fromValues(Expression... values) { + return tableEnv.fromValues(values); } @Override - public void useModules(String... moduleNames) { - tableEnv.useModules(moduleNames); + public Table fromValues(AbstractDataType rowType, Expression... values) { + return tableEnv.fromValues(rowType, values); } @Override - public void createFunction( - String path, String className, List resourceUris) { - tableEnv.createFunction(path, className, resourceUris); + public Table fromValues(Iterable values) { + return tableEnv.fromValues(values); } @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - tableEnv.createFunction(path, className, resourceUris, ignoreIfExists); + public Table fromValues(AbstractDataType rowType, Iterable values) { + return tableEnv.fromValues(rowType, values); } @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - tableEnv.createTemporaryFunction(path, className, resourceUris); + public void registerCatalog(String catalogName, Catalog catalog) { + tableEnv.registerCatalog(catalogName, catalog); } @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - tableEnv.createTemporarySystemFunction(name, className, resourceUris); + public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { + tableEnv.createCatalog(catalogName, catalogDescriptor); } @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - tableEnv.createTemporaryTable(path, descriptor); + public Optional getCatalog(String catalogName) { + return tableEnv.getCatalog(catalogName); } @Override - public void createTable(String path, TableDescriptor descriptor) { - tableEnv.createTable(path, descriptor); + public void loadModule(String moduleName, Module module) { + tableEnv.loadModule(moduleName, module); } @Override - public Table from(TableDescriptor descriptor) { - return tableEnv.from(descriptor); + public void useModules(String... moduleNames) { + tableEnv.useModules(moduleNames); } @Override - public ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); + public void unloadModule(String moduleName) { + tableEnv.unloadModule(moduleName); } @Override - public String[] listTables(String catalogName, String databaseName) { - return tableEnv.listTables(catalogName, databaseName); + public void registerFunction(String name, ScalarFunction function) { + tableEnv.registerFunction(name, function); } @Override - public String explainSql( - String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, format, extraDetails); + public void createTemporarySystemFunction(String name, Class functionClass) { + tableEnv.createTemporarySystemFunction(name, functionClass); } @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return tableEnv.loadPlan(planReference); + public void createTemporarySystemFunction(String name, UserDefinedFunction functionInstance) { + tableEnv.createTemporarySystemFunction(name, functionInstance); } @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return tableEnv.compilePlanSql(statement); + public boolean dropTemporarySystemFunction(String name) { + return tableEnv.dropTemporarySystemFunction(name); } @Override - public void registerCatalog(String catalogName, Catalog catalog) { - tableEnv.registerCatalog(catalogName, catalog); + public void createFunction(String path, Class functionClass) { + tableEnv.createFunction(path, functionClass); } @Override - public Optional getCatalog(String catalogName) { - return tableEnv.getCatalog(catalogName); + public void createFunction(String path, Class functionClass, + boolean ignoreIfExists) { + tableEnv.createFunction(path, functionClass, ignoreIfExists); } @Override - public void loadModule(String moduleName, Module module) { - tableEnv.loadModule(moduleName, module); + public void createFunction(String path, String className, List resourceUris) { + tableEnv.createFunction(path, className, resourceUris); } @Override - public void unloadModule(String moduleName) { - tableEnv.unloadModule(moduleName); + public void createFunction(String path, String className, List resourceUris, boolean ignoreIfExists) { + tableEnv.createFunction(path, className, resourceUris, ignoreIfExists); } @Override - public void createTemporarySystemFunction( - String name, Class functionClass) { - tableEnv.createTemporarySystemFunction(name, functionClass); + public void createTemporaryFunction(String path, Class functionClass) { + tableEnv.createTemporaryFunction(path, functionClass); } @Override - public void createTemporarySystemFunction( - String name, UserDefinedFunction functionInstance) { - tableEnv.createTemporarySystemFunction(name, functionInstance); + public void createTemporaryFunction(String path, UserDefinedFunction functionInstance) { + tableEnv.createTemporaryFunction(path, functionInstance); } @Override - public boolean dropTemporarySystemFunction(String name) { - return tableEnv.dropTemporarySystemFunction(name); + public void createTemporaryFunction(String path, String className, List resourceUris) { + tableEnv.createTemporaryFunction(path, className, resourceUris); } @Override - public void createFunction(String path, Class functionClass) { - tableEnv.createFunction(path, functionClass); + public void createTemporarySystemFunction(String name, String className, List resourceUris) { + tableEnv.createTemporarySystemFunction(name, className, resourceUris); } @Override - public void createFunction( - String path, - Class functionClass, - boolean ignoreIfExists) { - tableEnv.createFunction(path, functionClass, ignoreIfExists); + public boolean dropFunction(String path) { + return tableEnv.dropFunction(path); } @Override - public boolean dropFunction(String path) { - return tableEnv.dropFunction(path); + public boolean dropTemporaryFunction(String path) { + return tableEnv.dropTemporaryFunction(path); } @Override - public void createTemporaryFunction( - String path, Class functionClass) { - tableEnv.createTemporaryFunction(path, functionClass); + public void createTemporaryTable(String path, TableDescriptor descriptor) { + tableEnv.createTemporaryTable(path, descriptor); } @Override - public void createTemporaryFunction(String path, UserDefinedFunction functionInstance) { - tableEnv.createTemporaryFunction(path, functionInstance); + public void createTable(String path, TableDescriptor descriptor) { + tableEnv.createTable(path, descriptor); } @Override - public boolean dropTemporaryFunction(String path) { - return tableEnv.dropTemporaryFunction(path); + public void registerTable(String name, Table table) { + tableEnv.registerTable(name, table); } @Override @@ -249,11 +247,22 @@ public void createTemporaryView(String path, Table view) { tableEnv.createTemporaryView(path, view); } + @Override + @Deprecated + public Table scan(String... tablePath) { + return tableEnv.scan(tablePath); + } + @Override public Table from(String path) { return tableEnv.from(path); } + @Override + public Table from(TableDescriptor descriptor) { + return tableEnv.from(descriptor); + } + @Override public String[] listCatalogs() { return tableEnv.listCatalogs(); @@ -264,6 +273,11 @@ public String[] listModules() { return tableEnv.listModules(); } + @Override + public ModuleEntry[] listFullModules() { + return tableEnv.listFullModules(); + } + @Override public String[] listDatabases() { return tableEnv.listDatabases(); @@ -274,6 +288,11 @@ public String[] listTables() { return tableEnv.listTables(); } + @Override + public String[] listTables(String catalogName, String databaseName) { + return tableEnv.listTables(catalogName, databaseName); + } + @Override public String[] listViews() { return tableEnv.listViews(); @@ -310,8 +329,13 @@ public boolean dropTemporaryView(String path) { } @Override - public String explainSql(String statement, ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, extraDetails); + public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { + return tableEnv.explainSql(statement, format, extraDetails); + } + + @Override + public String[] getCompletionHints(String statement, int position) { + return tableEnv.getCompletionHints(statement, position); } @Override @@ -330,7 +354,7 @@ public String getCurrentCatalog() { } @Override - public void useCatalog(String catalogName) { + public void useCatalog(@Nullable String catalogName) { tableEnv.useCatalog(catalogName); } @@ -340,7 +364,7 @@ public String getCurrentDatabase() { } @Override - public void useDatabase(String databaseName) { + public void useDatabase(@Nullable String databaseName) { tableEnv.useDatabase(databaseName); } @@ -355,28 +379,12 @@ public StatementSet createStatementSet() { } @Override - public String[] getCompletionHints(String statement, int position) { - return tableEnv.getCompletionHints(statement, position); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public Table scan(String... tablePath) { - return tableEnv.scan(tablePath); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void registerTable(String name, Table table) { - tableEnv.registerTable(name, table); + public CompiledPlan loadPlan(PlanReference planReference) throws TableException { + return tableEnv.loadPlan(planReference); } - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) @Override - public void registerFunction(String name, ScalarFunction function) { - tableEnv.registerFunction(name, function); + public CompiledPlan compilePlanSql(String stmt) throws TableException { + return tableEnv.compilePlanSql(stmt); } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkParameterUtils.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkParameterUtils.java deleted file mode 100644 index d1868525ce..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkParameterUtils.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.util.AssertUtils; -import org.apache.streampark.common.util.DeflaterUtils; - -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.configuration.PipelineOptions; -import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; - -import javax.annotation.Nullable; - -/** Parameter and table environment helpers for Flink application naming. */ -public final class FlinkParameterUtils { - - private FlinkParameterUtils() { - } - - public static String getAppName(ParameterTool parameterTool) { - return getAppName(parameterTool, null, false); - } - - public static String getAppName(ParameterTool parameterTool, boolean required) { - return getAppName(parameterTool, null, required); - } - - public static String getAppName( - ParameterTool parameterTool, @Nullable String name, boolean required) { - String appName; - if (name == null) { - appName = null; - String zippedAppName = parameterTool.get(ConfigKeys.KEY_APP_NAME(), null); - if (zippedAppName != null) { - try { - appName = DeflaterUtils.unzipString(zippedAppName); - } catch (Exception ignored) { - // fall back to pipeline.name - } - } - if (appName == null) { - appName = parameterTool.get(ConfigKeys.KEY_FLINK_APP_NAME(), null); - } - } else { - appName = name; - } - if (required) { - AssertUtils.required( - appName != null, "[StreamPark] Application name cannot be null"); - } - return appName; - } - - public static T setAppName(T env, ParameterTool parameter) { - String appName = getAppName(parameter); - if (appName != null) { - env.getConfig().getConfiguration().setString(PipelineOptions.NAME, appName); - } - return env; - } - - public static StreamTableEnvironment setAppName( - StreamTableEnvironment env, ParameterTool parameter) { - String appName = getAppName(parameter); - if (appName != null) { - env.getConfig().getConfiguration().setString(PipelineOptions.NAME, appName); - } - return env; - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java index 278f40a5b3..874253b442 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkSqlExecutor.java @@ -17,24 +17,24 @@ package org.apache.streampark.flink.core; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.option.ApplicationOptions; import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.StreamParkLoggerFactory; +import org.apache.streampark.flink.configuration.FlinkJobParameters; import org.apache.streampark.shaded.org.slf4j.Logger; import org.apache.commons.lang3.StringUtils; -import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.ExecutionOptions; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.TableResult; -import org.apache.flink.table.api.TableSchema; +import org.apache.flink.table.catalog.ResolvedSchema; import org.apache.flink.types.Row; import java.util.Arrays; import java.util.EnumMap; -import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -56,17 +56,29 @@ public final class FlinkSqlExecutor { private FlinkSqlExecutor() { } - public static void executeSql(String sql, ParameterTool parameter, TableEnvironment context) { - executeSql(sql, parameter, context, null); + /** Executes SQL configured by the default application SQL key. */ + public static void executeSql( + FlinkJobParameters parameter, + TableEnvironment context) { + executeSql(null, parameter, context, null); + } + + /** Executes SQL configured by the supplied parameter key. */ + public static void executeSql( + String sqlKey, + FlinkJobParameters parameter, + TableEnvironment context) { + executeSql(sqlKey, parameter, context, null); } + /** Executes SQL configured by the supplied parameter key and reports metadata to a callback. */ public static void executeSql( - String sql, - ParameterTool parameter, + String sqlKey, + FlinkJobParameters parameter, TableEnvironment context, - Consumer callbackFunc) { - String flinkSql = resolveSql(sql, parameter); - ExecutionContext ctx = new ExecutionContext(context, callbackFunc, parameter); + Consumer callback) { + String flinkSql = resolveSql(sqlKey, parameter); + ExecutionContext ctx = new ExecutionContext(context, parameter, callback); List calls = SqlCommandParser.parseSQL(flinkSql, null); for (SqlCommandCall call : calls) { processCommand(call, ctx); @@ -74,11 +86,11 @@ public static void executeSql( finishExecution(flinkSql, ctx); } - private static String resolveSql(String sql, ParameterTool parameter) { + private static String resolveSql(String sqlKey, FlinkJobParameters parameter) { String flinkSql = - StringUtils.isBlank(sql) - ? parameter.get(ConfigKeys.KEY_FLINK_SQL()) - : parameter.get(sql); + StringUtils.isBlank(sqlKey) + ? parameter.get(ApplicationOptions.SQL) + : parameter.get(sqlKey); if (StringUtils.isBlank(flinkSql)) { throw new IllegalArgumentException("verify failed: flink sql cannot be empty"); } @@ -163,16 +175,20 @@ private static void showTables(SqlCommandCall call, ExecutionContext ctx) { private static void describeTable(SqlCommandCall call, ExecutionContext ctx) { String args = firstOperand(call); - TableSchema schema = ctx.context.scan(args).getSchema(); + ResolvedSchema schema = ctx.context.from(args).getResolvedSchema(); + ctx.callback.accept(formatSchema(schema)); + } + + static String formatSchema(ResolvedSchema schema) { StringBuilder builder = new StringBuilder(); builder.append("Column\tType\n"); - for (int i = 0; i <= schema.getFieldCount(); i++) { - builder.append(schema.getFieldName(i).get()) + for (int i = 0; i < schema.getColumnCount(); i++) { + builder.append(schema.getColumnNames().get(i)) .append("\t") - .append(schema.getFieldDataType(i).get()) + .append(schema.getColumnDataTypes().get(i)) .append("\n"); } - ctx.callback.accept(builder.toString()); + return builder.toString(); } private static void explainSql(SqlCommandCall call, ExecutionContext ctx) { @@ -193,24 +209,15 @@ private static void setConfig(SqlCommandCall call, ExecutionContext ctx) { private static void resetConfig(SqlCommandCall call, ExecutionContext ctx) { String args = firstOperand(call); - try { - java.lang.reflect.Field confDataField = - Configuration.class.getDeclaredField("confData"); - confDataField.setAccessible(true); - @SuppressWarnings("unchecked") - HashMap confData = - (HashMap) confDataField.get(ctx.context.getConfig().getConfiguration()); - synchronized (confData) { - if (call.command == SqlCommand.RESET) { - confData.remove(args); - } else { - confData.clear(); - } + Configuration configuration = ctx.context.getConfig().getConfiguration(); + if (call.command == SqlCommand.RESET) { + configuration.removeKey(args); + } else { + for (String key : new HashSet<>(configuration.keySet())) { + configuration.removeKey(key); } - LOG.info("{}: {}", call.command.getName(), args); - } catch (ReflectiveOperationException e) { - throw new IllegalStateException("Failed to reset Flink table configuration", e); } + LOG.info("{}: {}", call.command.getName(), args); } private static void warnStatementSet(SqlCommandCall call, ExecutionContext ctx) { @@ -270,24 +277,18 @@ private static final class ExecutionContext { private final TableEnvironment context; private final Consumer callback; - private final ParameterTool parameter; + private final FlinkJobParameters parameter; private final org.apache.flink.table.api.StatementSet statementSet; private boolean hasInsert; private ExecutionContext( - TableEnvironment context, Consumer callbackFunc, - ParameterTool parameter) { + TableEnvironment context, + FlinkJobParameters parameter, + Consumer callback) { this.context = context; this.parameter = parameter; this.statementSet = context.createStatementSet(); - this.callback = - r -> { - if (callbackFunc != null) { - callbackFunc.accept(r); - } else { - LOG.info(r); - } - }; + this.callback = callback == null ? LOG::info : callback; } } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamInitializer.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamInitializer.java new file mode 100644 index 0000000000..7729e69f21 --- /dev/null +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamInitializer.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.core; + +import org.apache.streampark.common.configuration.CommandLineParser; +import org.apache.streampark.common.configuration.ConfigException; +import org.apache.streampark.common.configuration.Configuration; +import org.apache.streampark.common.configuration.ConfigurationFormat; +import org.apache.streampark.common.configuration.ConfigurationParser; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.option.ApplicationOptions; +import org.apache.streampark.common.util.DeflaterUtils; +import org.apache.streampark.common.util.HdfsUtils; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.configuration.FlinkRuntimeConfiguration; +import org.apache.streampark.flink.core.bean.StreamContextSpec; + +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Locale; + +/** + * Builds the configuration and execution environment for a Flink streaming application. + * + *

Application files are split into explicit namespaces. {@code flink.property.*} becomes native + * Flink configuration, {@code app.*} becomes user application parameters, and command-line values + * override both. This is the only place where those source layers are assembled. + */ +public class FlinkStreamInitializer { + + final String[] args; + + private FlinkRuntimeConfiguration configuration; + private StreamExecutionEnvironment streamEnv; + + FlinkStreamInitializer(String[] args) { + this.args = args.clone(); + } + + public static StreamContextSpec initializeStream(String[] args) { + FlinkStreamInitializer initializer = new FlinkStreamInitializer(args); + return new StreamContextSpec( + initializer.getConfiguration().jobParameters(), initializer.getStreamEnv()); + } + + FlinkJobParameters getParameter() { + return getConfiguration().jobParameters(); + } + + FlinkRuntimeConfiguration getConfiguration() { + if (configuration == null) { + configuration = initParameter(); + } + return configuration; + } + + StreamExecutionEnvironment getStreamEnv() { + if (streamEnv == null) { + streamEnv = + StreamExecutionEnvironment.getExecutionEnvironment( + getConfiguration().environmentConfiguration()); + streamEnv.getConfig().setGlobalJobParameters(getParameter()); + } + return streamEnv; + } + + FlinkRuntimeConfiguration initParameter() { + Configuration arguments = CommandLineParser.parse(args); + String configResource = + arguments + .getOptional(ApplicationOptions.CONFIG) + .orElseThrow( + () -> new ConfigException( + "Application configuration is required; use --" + + ApplicationOptions.CONFIG.key() + + " ")); + + Configuration document = parseConfig(configResource); + Configuration flinkProperties = document.subset(FlinkOptions.PROPERTY_PREFIX.value()); + Configuration applicationProperties = + document.subset(ApplicationOptions.APPLICATION_PREFIX.value()); + + // Native properties are also visible as job parameters for keys such as pipeline.name. + Configuration applicationConfiguration = + Configuration.builder() + .add(flinkProperties) + .add(applicationProperties) + .add(arguments) + .build(); + org.apache.flink.configuration.Configuration flinkConfiguration = + org.apache.flink.configuration.Configuration.fromMap(flinkProperties.toMap()); + return new FlinkRuntimeConfiguration( + applicationConfiguration, flinkConfiguration, null); + } + + /** Parses an inline, local, or HDFS application configuration resource. */ + Configuration parseConfig(String resource) { + if (resource.startsWith("yaml://")) { + return parseCompressed(resource, "yaml://", ConfigurationFormat.YAML); + } + if (resource.startsWith("conf://")) { + return parseCompressed(resource, "conf://", ConfigurationFormat.HOCON); + } + if (resource.startsWith("prop://")) { + return parseCompressed(resource, "prop://", ConfigurationFormat.PROPERTIES); + } + if (resource.startsWith("hdfs://")) { + try { + return ConfigurationParser.parse( + HdfsUtils.read(resource), formatOf(resource), resource); + } catch (IOException e) { + throw new ConfigException( + "Cannot read application configuration from HDFS: " + resource, e); + } + } + return ConfigurationParser.parse(Path.of(resource)); + } + + private static Configuration parseCompressed( + String resource, + String scheme, + ConfigurationFormat format) { + try { + String content = DeflaterUtils.unzipString(resource.substring(scheme.length())); + return ConfigurationParser.parse(content, format, "inline " + format.name().toLowerCase(Locale.ROOT)); + } catch (RuntimeException e) { + throw new ConfigException("Cannot decode inline " + format + " configuration", e); + } + } + + private static ConfigurationFormat formatOf(String resource) { + String normalized = resource.toLowerCase(Locale.ROOT); + if (normalized.endsWith(".yaml") || normalized.endsWith(".yml")) { + return ConfigurationFormat.YAML; + } + if (normalized.endsWith(".conf") || normalized.endsWith(".hocon")) { + return ConfigurationFormat.HOCON; + } + if (normalized.endsWith(".properties")) { + return ConfigurationFormat.PROPERTIES; + } + throw new ConfigException( + "Application configuration must be YAML, HOCON, or properties: " + resource); + } +} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTrait.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTrait.java deleted file mode 100644 index 8522266214..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamTableTrait.java +++ /dev/null @@ -1,845 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.streampark.common.util.Utils; - -import org.apache.flink.api.common.JobExecutionResult; -import org.apache.flink.api.common.RuntimeExecutionMode; -import org.apache.flink.api.common.cache.DistributedCache; -import org.apache.flink.api.common.eventtime.WatermarkStrategy; -import org.apache.flink.api.common.io.FileInputFormat; -import org.apache.flink.api.common.io.FilePathFilter; -import org.apache.flink.api.common.io.InputFormat; -import org.apache.flink.api.common.restartstrategy.RestartStrategies; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.connector.source.Source; -import org.apache.flink.api.connector.source.SourceSplit; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.configuration.ReadableConfig; -import org.apache.flink.core.execution.JobClient; -import org.apache.flink.core.execution.JobListener; -import org.apache.flink.runtime.state.StateBackend; -import org.apache.flink.streaming.api.CheckpointingMode; -import org.apache.flink.streaming.api.TimeCharacteristic; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.CheckpointConfig; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.streaming.api.functions.source.FileMonitoringFunction; -import org.apache.flink.streaming.api.functions.source.SourceFunction; -import org.apache.flink.streaming.api.graph.StreamGraph; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.Schema; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableConfig; -import org.apache.flink.table.api.TableDescriptor; -import org.apache.flink.table.api.TableException; -import org.apache.flink.table.api.TableResult; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.table.catalog.Catalog; -import org.apache.flink.table.catalog.CatalogDescriptor; -import org.apache.flink.table.connector.ChangelogMode; -import org.apache.flink.table.expressions.Expression; -import org.apache.flink.table.functions.AggregateFunction; -import org.apache.flink.table.functions.ScalarFunction; -import org.apache.flink.table.functions.TableAggregateFunction; -import org.apache.flink.table.functions.TableFunction; -import org.apache.flink.table.functions.UserDefinedFunction; -import org.apache.flink.table.module.Module; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; -import org.apache.flink.table.types.AbstractDataType; -import org.apache.flink.types.Row; -import org.apache.flink.util.SplittableIterator; - -import com.esotericsoftware.kryo.Serializer; - -import java.io.Serializable; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import java.util.function.Consumer; - -/** - * Integration API of stream and table environments. - * - *

Once a Table has been converted to a DataStream, the DataStream job must be executed using the - * execute method of the StreamExecutionEnvironment. - */ -@SuppressWarnings("java:S100") -public abstract class FlinkStreamTableTrait implements StreamTableEnvironment { - - public final ParameterTool parameter; - - private final StreamExecutionEnvironment streamEnv; - - private final StreamTableEnvironment tableEnv; - - /** Whether a table has been converted to a DataStream. */ - public boolean isConvertedToDataStream; - - protected FlinkStreamTableTrait( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment tableEnv) { - this.parameter = parameter; - this.streamEnv = streamEnv; - this.tableEnv = tableEnv; - } - - protected StreamExecutionEnvironment getStreamEnv() { - return streamEnv; - } - - protected StreamTableEnvironment getStreamTableEnv() { - return tableEnv; - } - - /** Recommended API to start tasks. */ - public JobExecutionResult start() { - return start(null); - } - - public JobExecutionResult start(String name) { - String appName = FlinkParameterUtils.getAppName(parameter, name, true); - return execute(appName); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - public JobExecutionResult execute(String jobName) { - Utils.printLogo("FlinkStreamTable " + jobName + " Starting..."); - if (isConvertedToDataStream) { - try { - return streamEnv.execute(jobName); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - return null; - } - - public void sql(String sql) { - sql(sql, null); - } - - public void sql(String sql, Consumer callback) { - FlinkSqlExecutor.executeSql(sql, parameter, this, callback); - } - - public StreamExecutionEnvironment getJavaEnv() { - return streamEnv; - } - - public List> $getCachedFiles() { - return streamEnv.getCachedFiles(); - } - - public List $getJobListeners() { - return streamEnv.getJobListeners(); - } - - public void $setParallelism(int parallelism) { - streamEnv.setParallelism(parallelism); - } - - public StreamExecutionEnvironment $setRuntimeMode(RuntimeExecutionMode deployMode) { - return streamEnv.setRuntimeMode(deployMode); - } - - public void $setMaxParallelism(int maxParallelism) { - streamEnv.setMaxParallelism(maxParallelism); - } - - public int $getParallelism() { - return streamEnv.getParallelism(); - } - - public int $getMaxParallelism() { - return streamEnv.getMaxParallelism(); - } - - public StreamExecutionEnvironment $setBufferTimeout(long timeoutMillis) { - return streamEnv.setBufferTimeout(timeoutMillis); - } - - public long $getBufferTimeout() { - return streamEnv.getBufferTimeout(); - } - - public StreamExecutionEnvironment $disableOperatorChaining() { - return streamEnv.disableOperatorChaining(); - } - - public CheckpointConfig $getCheckpointConfig() { - return streamEnv.getCheckpointConfig(); - } - - public StreamExecutionEnvironment $enableCheckpointing(long interval, CheckpointingMode mode) { - return streamEnv.enableCheckpointing(interval, mode); - } - - public StreamExecutionEnvironment $enableCheckpointing(long interval) { - return streamEnv.enableCheckpointing(interval); - } - - public CheckpointingMode $getCheckpointingMode() { - return streamEnv.getCheckpointConfig().getCheckpointingMode(); - } - - public StreamExecutionEnvironment $setStateBackend(StateBackend backend) { - return streamEnv.setStateBackend(backend); - } - - public StateBackend $getStateBackend() { - return streamEnv.getStateBackend(); - } - - public void $setRestartStrategy( - RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration) { - streamEnv.setRestartStrategy(restartStrategyConfiguration); - } - - public RestartStrategies.RestartStrategyConfiguration $getRestartStrategy() { - return streamEnv.getRestartStrategy(); - } - - public void $setNumberOfExecutionRetries(int numRetries) { - streamEnv.setNumberOfExecutionRetries(numRetries); - } - - public int $getNumberOfExecutionRetries() { - return streamEnv.getNumberOfExecutionRetries(); - } - - public & Serializable> void $addDefaultKryoSerializer( - Class type, T serializer) { - streamEnv.addDefaultKryoSerializer(type, serializer); - } - - public void $addDefaultKryoSerializer( - Class type, Class> serializerClass) { - streamEnv.addDefaultKryoSerializer(type, serializerClass); - } - - public & Serializable> void $registerTypeWithKryoSerializer( - Class clazz, T serializer) { - streamEnv.registerTypeWithKryoSerializer(clazz, serializer); - } - - public void $registerTypeWithKryoSerializer( - Class clazz, Class> serializer) { - streamEnv.registerTypeWithKryoSerializer(clazz, serializer); - } - - public void $registerType(Class typeClass) { - streamEnv.registerType(typeClass); - } - - public TimeCharacteristic $getStreamTimeCharacteristic() { - return streamEnv.getStreamTimeCharacteristic(); - } - - public void $configure(ReadableConfig configuration, ClassLoader classLoader) { - streamEnv.configure(configuration, classLoader); - } - - public DataStream $fromSequence(long from, long to) { - return streamEnv.fromSequence(from, to); - } - - public DataStream $fromElements(T... data) { - return streamEnv.fromElements(data); - } - - public DataStream $fromCollection(Collection data) { - return streamEnv.fromCollection(data); - } - - public DataStream $fromCollection(Iterator data) { - java.util.List list = new java.util.ArrayList<>(); - data.forEachRemaining(list::add); - return streamEnv.fromCollection(list); - } - - public DataStream $fromParallelCollection( - SplittableIterator data, TypeInformation typeInfo) { - return streamEnv.fromParallelCollection(data, typeInfo); - } - - public DataStream $readTextFile(String filePath) { - return streamEnv.readTextFile(filePath); - } - - public DataStream $readTextFile(String filePath, String charsetName) { - return streamEnv.readTextFile(filePath, charsetName); - } - - public DataStream $readFile(FileInputFormat inputFormat, String filePath) { - return streamEnv.readFile(inputFormat, filePath); - } - - public DataStream $readFile( - FileInputFormat inputFormat, - String filePath, - org.apache.flink.streaming.api.functions.source.FileProcessingMode watchType, - long interval) { - return streamEnv.readFile(inputFormat, filePath, watchType, interval); - } - - public DataStream $socketTextStream( - String hostname, int port, char delimiter, long maxRetry) { - return streamEnv.socketTextStream(hostname, port, delimiter, maxRetry); - } - - public DataStream $createInput(InputFormat inputFormat) { - return streamEnv.createInput(inputFormat); - } - - public DataStream $addSource(SourceFunction function) { - return streamEnv.addSource(function); - } - - public DataStream $fromSource( - Source source, - WatermarkStrategy watermarkStrategy, - String sourceName) { - return streamEnv.fromSource(source, watermarkStrategy, sourceName); - } - - public void $registerJobListener(JobListener jobListener) { - streamEnv.registerJobListener(jobListener); - } - - public void $clearJobListeners() { - streamEnv.clearJobListeners(); - } - - public JobClient $executeAsync() throws Exception { - return streamEnv.executeAsync(); - } - - public JobClient $executeAsync(String jobName) throws Exception { - return streamEnv.executeAsync(jobName); - } - - public String $getExecutionPlan() { - return streamEnv.getExecutionPlan(); - } - - public StreamGraph $getStreamGraph() { - return streamEnv.getStreamGraph(); - } - - public StreamExecutionEnvironment $getWrappedStreamExecutionEnvironment() { - return streamEnv; - } - - public void $registerCachedFile(String filePath, String name) { - streamEnv.registerCachedFile(filePath, name); - } - - public void $registerCachedFile(String filePath, String name, boolean executable) { - streamEnv.registerCachedFile(filePath, name, executable); - } - - public boolean $isUnalignedCheckpointsEnabled() { - return streamEnv.getCheckpointConfig().isUnalignedCheckpointsEnabled(); - } - - public boolean $isForceUnalignedCheckpoints() { - return streamEnv.getCheckpointConfig().isForceUnalignedCheckpoints(); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - public StreamExecutionEnvironment $enableCheckpointing( - long interval, CheckpointingMode mode, boolean force) { - return streamEnv.enableCheckpointing(interval, mode); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - public DataStream $generateSequence(long from, long to) { - return streamEnv.fromSequence(from, to); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - public DataStream $readFileStream( - String streamPath, - long intervalMillis, - FileMonitoringFunction.WatchType watchType) { - return streamEnv.readFileStream(streamPath, intervalMillis, watchType); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - public DataStream $readFile( - FileInputFormat inputFormat, - String filePath, - org.apache.flink.streaming.api.functions.source.FileProcessingMode watchType, - long interval, - FilePathFilter filter) { - return streamEnv.readFile(inputFormat, filePath, watchType, interval, filter); - } - - @Override - public Table fromDataStream(DataStream dataStream, Schema schema) { - return tableEnv.fromDataStream(dataStream, schema); - } - - @Override - public Table fromChangelogStream(DataStream dataStream) { - return tableEnv.fromChangelogStream(dataStream); - } - - @Override - public Table fromChangelogStream(DataStream dataStream, Schema schema) { - return tableEnv.fromChangelogStream(dataStream, schema); - } - - @Override - public Table fromChangelogStream( - DataStream dataStream, Schema schema, ChangelogMode changelogMode) { - return tableEnv.fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView(String path, DataStream dataStream, Schema schema) { - tableEnv.createTemporaryView(path, dataStream, schema); - } - - @Override - public DataStream toDataStream(Table table) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table); - } - - @Override - public DataStream toDataStream(Table table, Class targetClass) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetClass); - } - - @Override - public DataStream toDataStream(Table table, AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return tableEnv.toDataStream(table, targetDataType); - } - - @Override - public DataStream toChangelogStream(Table table) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table); - } - - @Override - public DataStream toChangelogStream(Table table, Schema targetSchema) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema); - } - - @Override - public DataStream toChangelogStream( - Table table, Schema targetSchema, ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return tableEnv.toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public DataStream toAppendStream(Table table, TypeInformation typeInfo) { - isConvertedToDataStream = true; - return tableEnv.toAppendStream(table, typeInfo); - } - - @Override - public DataStream> toRetractStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return tableEnv.toRetractStream(table, clazz); - } - - @Override - public DataStream> toRetractStream( - Table table, TypeInformation typeInfo) { - isConvertedToDataStream = true; - return tableEnv.toRetractStream(table, typeInfo); - } - - @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - tableEnv.createCatalog(catalogName, catalogDescriptor); - } - - @Override - public void useModules(String... moduleNames) { - tableEnv.useModules(moduleNames); - } - - @Override - public void createFunction( - String path, String className, List resourceUris) { - tableEnv.createFunction(path, className, resourceUris); - } - - @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - tableEnv.createFunction(path, className, resourceUris, ignoreIfExists); - } - - @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - tableEnv.createTemporaryFunction(path, className, resourceUris); - } - - @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - tableEnv.createTemporarySystemFunction(name, className, resourceUris); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - tableEnv.createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - tableEnv.createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor descriptor) { - return tableEnv.from(descriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); - } - - @Override - public String[] listTables(String catalogName, String databaseName) { - return tableEnv.listTables(catalogName, databaseName); - } - - @Override - public String explainSql( - String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, format, extraDetails); - } - - @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return tableEnv.loadPlan(planReference); - } - - @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return tableEnv.compilePlanSql(statement); - } - - @Override - public Table fromDataStream(DataStream dataStream) { - return tableEnv.fromDataStream(dataStream); - } - - @Override - public Table fromDataStream(DataStream dataStream, Expression... fields) { - return tableEnv.fromDataStream(dataStream, fields); - } - - @Override - public void createTemporaryView(String path, DataStream dataStream) { - tableEnv.createTemporaryView(path, dataStream); - } - - @Override - public void createTemporaryView( - String path, DataStream dataStream, Expression... fields) { - tableEnv.createTemporaryView(path, dataStream, fields); - } - - @Override - public DataStream toAppendStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return tableEnv.toAppendStream(table, clazz); - } - - @Override - public Table fromValues(Expression... values) { - return tableEnv.fromValues(values); - } - - @Override - public Table fromValues(AbstractDataType rowType, Expression... values) { - return tableEnv.fromValues(rowType, values); - } - - @Override - public Table fromValues(Iterable values) { - return tableEnv.fromValues(values); - } - - @Override - public Table fromValues(AbstractDataType rowType, Iterable values) { - return tableEnv.fromValues(rowType, values); - } - - @Override - public void registerCatalog(String catalogName, Catalog catalog) { - tableEnv.registerCatalog(catalogName, catalog); - } - - @Override - public Optional getCatalog(String catalogName) { - return tableEnv.getCatalog(catalogName); - } - - @Override - public void loadModule(String moduleName, Module module) { - tableEnv.loadModule(moduleName, module); - } - - @Override - public void unloadModule(String moduleName) { - tableEnv.unloadModule(moduleName); - } - - @Override - public void createTemporarySystemFunction( - String name, Class functionClass) { - tableEnv.createTemporarySystemFunction(name, functionClass); - } - - @Override - public void createTemporarySystemFunction( - String name, UserDefinedFunction functionInstance) { - tableEnv.createTemporarySystemFunction(name, functionInstance); - } - - @Override - public boolean dropTemporarySystemFunction(String name) { - return tableEnv.dropTemporarySystemFunction(name); - } - - @Override - public void createFunction(String path, Class functionClass) { - tableEnv.createFunction(path, functionClass); - } - - @Override - public void createFunction( - String path, - Class functionClass, - boolean ignoreIfExists) { - tableEnv.createFunction(path, functionClass, ignoreIfExists); - } - - @Override - public boolean dropFunction(String path) { - return tableEnv.dropFunction(path); - } - - @Override - public void createTemporaryFunction( - String path, Class functionClass) { - tableEnv.createTemporaryFunction(path, functionClass); - } - - @Override - public void createTemporaryFunction(String path, UserDefinedFunction functionInstance) { - tableEnv.createTemporaryFunction(path, functionInstance); - } - - @Override - public boolean dropTemporaryFunction(String path) { - return tableEnv.dropTemporaryFunction(path); - } - - @Override - public void createTemporaryView(String path, Table view) { - tableEnv.createTemporaryView(path, view); - } - - @Override - public Table from(String path) { - return tableEnv.from(path); - } - - @Override - public String[] listCatalogs() { - return tableEnv.listCatalogs(); - } - - @Override - public String[] listModules() { - return tableEnv.listModules(); - } - - @Override - public String[] listDatabases() { - return tableEnv.listDatabases(); - } - - @Override - public String[] listTables() { - return tableEnv.listTables(); - } - - @Override - public String[] listViews() { - return tableEnv.listViews(); - } - - @Override - public String[] listTemporaryTables() { - return tableEnv.listTemporaryTables(); - } - - @Override - public String[] listTemporaryViews() { - return tableEnv.listTemporaryViews(); - } - - @Override - public String[] listUserDefinedFunctions() { - return tableEnv.listUserDefinedFunctions(); - } - - @Override - public String[] listFunctions() { - return tableEnv.listFunctions(); - } - - @Override - public boolean dropTemporaryTable(String path) { - return tableEnv.dropTemporaryTable(path); - } - - @Override - public boolean dropTemporaryView(String path) { - return tableEnv.dropTemporaryView(path); - } - - @Override - public String explainSql(String statement, ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, extraDetails); - } - - @Override - public Table sqlQuery(String query) { - return tableEnv.sqlQuery(query); - } - - @Override - public TableResult executeSql(String statement) { - return tableEnv.executeSql(statement); - } - - @Override - public String getCurrentCatalog() { - return tableEnv.getCurrentCatalog(); - } - - @Override - public void useCatalog(String catalogName) { - tableEnv.useCatalog(catalogName); - } - - @Override - public String getCurrentDatabase() { - return tableEnv.getCurrentDatabase(); - } - - @Override - public void useDatabase(String databaseName) { - tableEnv.useDatabase(databaseName); - } - - @Override - public TableConfig getConfig() { - return tableEnv.getConfig(); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void registerFunction(String name, TableFunction function) { - tableEnv.registerFunction(name, function); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void registerFunction(String name, AggregateFunction function) { - tableEnv.registerFunction(name, function); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void registerFunction(String name, TableAggregateFunction function) { - tableEnv.registerFunction(name, function); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void registerDataStream(String name, DataStream dataStream) { - tableEnv.registerDataStream(name, dataStream); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void registerFunction(String name, ScalarFunction function) { - tableEnv.registerFunction(name, function); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void registerTable(String name, Table table) { - tableEnv.registerTable(name, table); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public Table scan(String... tablePath) { - return tableEnv.scan(tablePath); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public String[] getCompletionHints(String statement, int position) { - return tableEnv.getCompletionHints(statement, position); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializer.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializer.java deleted file mode 100644 index 8f232989fb..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkStreamingInitializer.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.FileUtils; -import org.apache.streampark.common.util.HdfsUtils; -import org.apache.streampark.common.util.PropertiesUtils; -import org.apache.streampark.flink.core.conf.FlinkConfiguration; - -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -/** Initializes Flink streaming execution environment from application arguments. */ -public class FlinkStreamingInitializer { - - final String[] args; - - StreamEnvConfigFunction javaStreamEnvConfFunc; - - private FlinkConfiguration configuration; - - private StreamExecutionEnvironment streamEnv; - - FlinkStreamingInitializer(String[] args) { - this.args = args; - } - - public static StreamingInitResult initialize(String[] args, StreamEnvConfigFunction config) { - FlinkStreamingInitializer flinkInitializer = new FlinkStreamingInitializer(args); - flinkInitializer.javaStreamEnvConfFunc = config; - return new StreamingInitResult( - flinkInitializer.getConfiguration().parameter, flinkInitializer.getStreamEnv()); - } - - public static StreamingInitResult initialize(StreamEnvConfig args) { - FlinkStreamingInitializer flinkInitializer = new FlinkStreamingInitializer(args.args); - flinkInitializer.javaStreamEnvConfFunc = args.conf; - return new StreamingInitResult( - flinkInitializer.getConfiguration().parameter, flinkInitializer.getStreamEnv()); - } - - ParameterTool getParameter() { - return getConfiguration().parameter; - } - - FlinkConfiguration getConfiguration() { - if (configuration == null) { - configuration = initParameter(); - } - return configuration; - } - - StreamExecutionEnvironment getStreamEnv() { - if (streamEnv == null) { - streamEnv = - StreamExecutionEnvironment.getExecutionEnvironment( - getConfiguration().envConfig); - if (javaStreamEnvConfFunc != null) { - javaStreamEnvConfFunc.configuration(streamEnv, getConfiguration().parameter); - } - streamEnv.getConfig().setGlobalJobParameters(getConfiguration().parameter); - } - return streamEnv; - } - - FlinkConfiguration initParameter() { - ParameterTool argsMap = ParameterTool.fromArgs(args); - String configFile = argsMap.get(ConfigKeys.KEY_APP_CONF(), null); - if (configFile == null || configFile.isEmpty()) { - throw new ExceptionInInitializerError( - "[StreamPark] Usage:can't find config,please set \"--conf $path \" in main arguments"); - } - Map configMap = parseConfig(configFile); - Map properConf = - extractConfigByPrefix(configMap, ConfigKeys.KEY_FLINK_PROPERTY_PREFIX()); - Map appConf = - extractConfigByPrefix(configMap, ConfigKeys.KEY_APP_PREFIX()); - - ParameterTool parameter = - ParameterTool.fromSystemProperties() - .mergeWith(ParameterTool.fromMap(properConf)) - .mergeWith(ParameterTool.fromMap(appConf)) - .mergeWith(argsMap); - - Configuration envConfig = Configuration.fromMap(properConf); - return new FlinkConfiguration(parameter, envConfig, null); - } - - Map parseConfig(String config) { - Map map; - if (config.startsWith("yaml://")) { - map = PropertiesUtils.fromYamlText(DeflaterUtils.unzipString(config.substring(7))); - } else if (config.startsWith("conf://")) { - map = PropertiesUtils.fromHoconText(DeflaterUtils.unzipString(config.substring(7))); - } else if (config.startsWith("prop://")) { - map = - PropertiesUtils.fromPropertiesText( - DeflaterUtils.unzipString(config.substring(7))); - } else if (config.startsWith("hdfs://")) { - try { - String text = HdfsUtils.read(config); - map = readConfig(config, text); - } catch (IOException e) { - throw new IllegalArgumentException( - "[StreamPark] Failed to read application config from HDFS: " + config, e); - } - } else { - File file = new File(config); - if (!file.exists()) { - throw new IllegalArgumentException( - "[StreamPark] Usage: application config file: " - + file - + " is not found!!!"); - } - try { - map = readConfig(config, FileUtils.readFile(file)); - } catch (IOException e) { - throw new IllegalArgumentException( - "[StreamPark] Failed to read application config file: " + config, e); - } - } - Map filtered = new HashMap<>(); - map.forEach( - (key, value) -> { - if (value != null && !value.isEmpty()) { - filtered.put(key, value); - } - }); - return filtered; - } - - private Map readConfig(String config, String text) { - String format = config.substring(config.lastIndexOf('.') + 1).toLowerCase(); - switch (format) { - case "yml": - case "yaml": - return PropertiesUtils.fromYamlText(text); - case "conf": - return PropertiesUtils.fromHoconText(text); - case "properties": - return PropertiesUtils.fromPropertiesText(text); - default: - throw new IllegalArgumentException( - "[StreamPark] Usage: application config file error,must be [yaml|conf|properties]"); - } - } - - Map extractConfigByPrefix(Map configMap, String prefix) { - Map map = new HashMap<>(); - configMap.forEach( - (key, value) -> { - if (key.startsWith(prefix)) { - map.put(key.substring(prefix.length()), value); - } - }); - return map; - } - - /** Streaming initialization result. */ - public static final class StreamingInitResult { - - public final ParameterTool parameter; - public final StreamExecutionEnvironment streamEnv; - - StreamingInitResult(ParameterTool parameter, StreamExecutionEnvironment streamEnv) { - this.parameter = parameter; - this.streamEnv = streamEnv; - } - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializer.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializer.java index 34c1e6b947..b731345e06 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializer.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializer.java @@ -17,75 +17,64 @@ package org.apache.streampark.flink.core; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.CommandLineParser; +import org.apache.streampark.common.configuration.ConfigException; +import org.apache.streampark.common.configuration.Configuration; +import org.apache.streampark.common.configuration.ConfigurationParser; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.option.ApplicationOptions; import org.apache.streampark.common.enums.PlannerType; import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.PropertiesUtils; +import org.apache.streampark.common.util.FlinkConfigurationUtils; import org.apache.streampark.common.util.StreamParkLoggerFactory; -import org.apache.streampark.flink.core.conf.FlinkConfiguration; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.configuration.FlinkRuntimeConfiguration; +import org.apache.streampark.flink.core.bean.StreamTableContextSpec; +import org.apache.streampark.flink.core.bean.TableContextSpec; +import org.apache.streampark.flink.util.FlinkParameterUtils; import org.apache.streampark.shaded.org.slf4j.Logger; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.configuration.Configuration; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; -import org.apache.flink.table.api.TableConfig; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import java.io.File; import java.lang.reflect.Method; -import java.util.HashMap; -import java.util.Map; +import java.nio.file.Path; -/** Initializes Flink table and stream-table environments from application arguments. */ -public class FlinkTableInitializer extends FlinkStreamingInitializer { +/** + * Builds batch-table and stream-table environments from namespaced application configuration. + * + *

Table configuration is retained separately from application parameters. The separation keeps + * keys such as {@code planner} and {@code catalog} from leaking into user job parameters while + * still allowing explicit command-line overrides through their full option names. + */ +public class FlinkTableInitializer extends FlinkStreamInitializer { private static final Logger LOG = - StreamParkLoggerFactory.loggerFactory() - .getLogger(FlinkTableInitializer.class.getName()); - - private TableEnvConfigFunction javaTableEnvConfFunc; + StreamParkLoggerFactory.loggerFactory().getLogger(FlinkTableInitializer.class.getName()); private EnvironmentSettings.Builder envSettingsBuilder; - private TableEnvironment tableEnv; - private StreamTableEnvironment streamTableEnv; FlinkTableInitializer(String[] args) { super(args); } - public static TableInitResult initialize(TableEnvConfig args) { - FlinkTableInitializer flinkInitializer = new FlinkTableInitializer(args.args); - flinkInitializer.javaTableEnvConfFunc = args.conf; - return new TableInitResult( - flinkInitializer.getConfiguration().parameter, flinkInitializer.getTableEnv()); - } - - public static StreamTableInitResult initialize(StreamTableEnvConfig args) { - FlinkTableInitializer flinkInitializer = new FlinkTableInitializer(args.args); - flinkInitializer.javaStreamEnvConfFunc = args.streamConfig; - flinkInitializer.javaTableEnvConfFunc = args.tableConfig; - return new StreamTableInitResult( - flinkInitializer.getConfiguration().parameter, - flinkInitializer.getStreamEnv(), - flinkInitializer.getStreamTableEnv()); + public static TableContextSpec initializeTable(String[] args) { + FlinkTableInitializer initializer = new FlinkTableInitializer(args); + return new TableContextSpec( + initializer.getConfiguration().jobParameters(), initializer.getTableEnv()); } - public static StreamTableInitResult initialize( - String[] args, - StreamEnvConfigFunction streamConfig, - TableEnvConfigFunction tableConfig) { - FlinkTableInitializer flinkInitializer = new FlinkTableInitializer(args); - flinkInitializer.javaStreamEnvConfFunc = streamConfig; - flinkInitializer.javaTableEnvConfFunc = tableConfig; - return new StreamTableInitResult( - flinkInitializer.getConfiguration().parameter, - flinkInitializer.getStreamEnv(), - flinkInitializer.getStreamTableEnv()); + public static StreamTableContextSpec initializeStreamTable(String[] args) { + FlinkTableInitializer initializer = new FlinkTableInitializer(args); + return new StreamTableContextSpec( + initializer.getConfiguration().jobParameters(), + initializer.getStreamEnv(), + initializer.getStreamTableEnv()); } TableEnvironment getTableEnv() { @@ -96,7 +85,6 @@ TableEnvironment getTableEnv() { tableEnv = FlinkParameterUtils.setAppName( TableEnvironment.create(builder.build()), getParameter()); - applyTableEnvConfig(tableEnv.getConfig()); } return tableEnv; } @@ -106,45 +94,32 @@ StreamTableEnvironment getStreamTableEnv() { LOG.info("components should work in streaming mode"); EnvironmentSettings.Builder builder = getEnvSettingsBuilder(); builder.inStreamingMode(); - EnvironmentSettings setting = builder.build(); - - if (javaStreamEnvConfFunc != null) { - javaStreamEnvConfFunc.configuration(getStreamEnv(), getParameter()); - } streamTableEnv = FlinkParameterUtils.setAppName( - StreamTableEnvironment.create(getStreamEnv(), setting), getParameter()); - applyTableEnvConfig(streamTableEnv.getConfig()); + StreamTableEnvironment.create(getStreamEnv(), builder.build()), + getParameter()); } return streamTableEnv; } - private void applyTableEnvConfig(TableConfig config) { - if (javaTableEnvConfFunc != null) { - javaTableEnvConfFunc.configuration(config, getParameter()); - } - } - private EnvironmentSettings.Builder getEnvSettingsBuilder() { if (envSettingsBuilder == null) { - envSettingsBuilder = buildEnvSettings(getParameter()); + envSettingsBuilder = buildEnvSettings(getConfiguration()); } return envSettingsBuilder; } - private EnvironmentSettings.Builder buildEnvSettings(ParameterTool parameter) { + private EnvironmentSettings.Builder buildEnvSettings( + FlinkRuntimeConfiguration runtimeConfiguration) { EnvironmentSettings.Builder builder = EnvironmentSettings.newInstance(); - - PlannerType plannerType = PlannerType.BLINK; - String plannerName = parameter.get(ConfigKeys.KEY_FLINK_TABLE_PLANNER(), null); - if (plannerName != null && !plannerName.isEmpty()) { - try { - plannerType = PlannerType.withName(plannerName); - } catch (IllegalArgumentException e) { - plannerType = PlannerType.BLINK; - } - } - + FlinkJobParameters parameters = runtimeConfiguration.jobParameters(); + Configuration tableConfiguration = tableConfiguration(runtimeConfiguration); + + String plannerName = + parameters + .getOptional(FlinkOptions.TABLE_PLANNER) + .orElseGet(() -> tableConfiguration.getOptionalString("planner").orElse(null)); + PlannerType plannerType = plannerType(plannerName); switch (plannerType) { case BLINK: invokePlannerMethod(builder, "useBlinkPlanner", "blinkPlanner will be used."); @@ -159,41 +134,72 @@ private EnvironmentSettings.Builder buildEnvSettings(ParameterTool parameter) { break; } - String flinkConf = parameter.get(ConfigKeys.KEY_FLINK_CONF(), null); - if (flinkConf == null || flinkConf.isEmpty()) { - throw new ExceptionInInitializerError( - "[StreamPark] Usage:can't find config,please set \"--flink.conf $conf \" in main arguments"); - } + String flinkConf = + parameters + .getOptional(FlinkOptions.FLINK_CONFIGURATION) + .orElseThrow( + () -> new ConfigException( + "Table applications require --" + + FlinkOptions.FLINK_CONFIGURATION.key())); + // Native Flink YAML requires Flink-specific list serialization and compatibility parsing; + // application YAML continues to use the engine-neutral parser. builder.withConfiguration( - Configuration.fromMap( - PropertiesUtils.fromYamlText(DeflaterUtils.unzipString(flinkConf)))); - - String catalog = parameter.get(ConfigKeys.KEY_FLINK_TABLE_CATALOG(), null); - String database = parameter.get(ConfigKeys.KEY_FLINK_TABLE_DATABASE(), null); - if (catalog != null && database != null) { - LOG.info("with built in catalog: {}", catalog); - LOG.info("with built in database: {}", database); - builder.withBuiltInCatalogName(catalog); - builder.withBuiltInDatabaseName(database); - } else if (catalog != null) { + org.apache.flink.configuration.Configuration.fromMap( + FlinkConfigurationUtils.loadConfigurationFromString( + DeflaterUtils.unzipString(flinkConf), + parameters.get(FlinkOptions.FLINK_CONFIGURATION_STANDARD_YAML)))); + + String catalog = + parameters + .getOptional(FlinkOptions.TABLE_CATALOG) + .orElseGet(() -> tableConfiguration.getOptionalString("catalog").orElse(null)); + String database = + parameters + .getOptional(FlinkOptions.TABLE_DATABASE) + .orElseGet(() -> tableConfiguration.getOptionalString("database").orElse(null)); + if (catalog != null) { LOG.info("with built in catalog: {}", catalog); builder.withBuiltInCatalogName(catalog); - } else if (database != null) { + } + if (database != null) { LOG.info("with built in database: {}", database); builder.withBuiltInDatabaseName(database); } return builder; } + private static PlannerType plannerType(String plannerName) { + if (plannerName == null || plannerName.isEmpty()) { + return PlannerType.BLINK; + } + try { + return PlannerType.withName(plannerName); + } catch (IllegalArgumentException ignored) { + return PlannerType.BLINK; + } + } + + private static Configuration tableConfiguration( + FlinkRuntimeConfiguration runtimeConfiguration) { + org.apache.flink.configuration.Configuration table = + runtimeConfiguration.tableConfiguration(); + return table == null + ? Configuration.empty() + : Configuration.builder() + .add("Flink table configuration", org.apache.streampark.common.configuration.ConfigSource.FILE, + table.toMap()) + .build(); + } + private void invokePlannerMethod( - EnvironmentSettings.Builder builder, String methodName, String successMessage) { + EnvironmentSettings.Builder builder, + String methodName, + String successMessage) { try { Method method = builder.getClass().getDeclaredMethod(methodName); method.setAccessible(true); method.invoke(builder); - if (successMessage != null) { - LOG.info(successMessage); - } + LOG.info(successMessage); } catch (NoSuchMethodException e) { LOG.warn("{} deprecated", methodName); } catch (ReflectiveOperationException e) { @@ -202,96 +208,63 @@ private void invokePlannerMethod( } @Override - FlinkConfiguration initParameter() { - ParameterTool argsMap = ParameterTool.fromArgs(args); - String configFile = argsMap.get(ConfigKeys.KEY_APP_CONF(), null); - FlinkConfiguration configuration; - if (configFile == null || configFile.isEmpty()) { - LOG.warn("Usage:can't find config,you can set \"--conf $path \" in main arguments"); - ParameterTool parameter = ParameterTool.fromSystemProperties().mergeWith(argsMap); - configuration = - new FlinkConfiguration(parameter, new Configuration(), new Configuration()); - } else { - Map configMap = parseConfig(configFile); - Map sqlConf = new HashMap<>(); - configMap.forEach( - (key, value) -> { - if (key.startsWith(ConfigKeys.KEY_SQL_PREFIX())) { - sqlConf.put(key.substring(ConfigKeys.KEY_SQL_PREFIX().length()), value); - } - }); - - Map properConf = - extractConfigByPrefix(configMap, ConfigKeys.KEY_FLINK_PROPERTY_PREFIX()); - Map appConf = - extractConfigByPrefix(configMap, ConfigKeys.KEY_APP_PREFIX()); - Map tableConf = - extractConfigByPrefix(configMap, ConfigKeys.KEY_FLINK_TABLE_PREFIX()); - - Configuration tableConfig = Configuration.fromMap(tableConf); - Configuration envConfig = Configuration.fromMap(properConf); - - ParameterTool parameter = - ParameterTool.fromSystemProperties() - .mergeWith(ParameterTool.fromMap(properConf)) - .mergeWith(ParameterTool.fromMap(tableConf)) - .mergeWith(ParameterTool.fromMap(appConf)) - .mergeWith(ParameterTool.fromMap(sqlConf)) - .mergeWith(argsMap); - - configuration = new FlinkConfiguration(parameter, envConfig, tableConfig); - } + FlinkRuntimeConfiguration initParameter() { + Configuration arguments = CommandLineParser.parse(args); + Configuration document = + arguments + .getOptional(FlinkOptions.APPLICATION_CONFIG) + .map(this::parseConfig) + .orElseGet(Configuration::empty); + + Configuration flinkProperties = document.subset(FlinkOptions.PROPERTY_PREFIX.value()); + Configuration tableProperties = document.subset(FlinkOptions.TABLE_PREFIX.value()); + Configuration applicationProperties = + document.subset(ApplicationOptions.APPLICATION_PREFIX.value()); + Configuration sqlProperties = document.subset(ApplicationOptions.SQL_PREFIX.value()); + + Configuration applicationConfiguration = + Configuration.builder() + .add(flinkProperties) + .add(applicationProperties) + .add(sqlProperties) + .add(arguments) + .build(); + FlinkRuntimeConfiguration assembled = + new FlinkRuntimeConfiguration( + applicationConfiguration, + org.apache.flink.configuration.Configuration.fromMap(flinkProperties.toMap()), + org.apache.flink.configuration.Configuration.fromMap(tableProperties.toMap())); + return resolveSql(assembled); + } - String flinkSql = configuration.parameter.get(ConfigKeys.KEY_FLINK_SQL(), null); - if (flinkSql == null) { + private FlinkRuntimeConfiguration resolveSql(FlinkRuntimeConfiguration configuration) { + String sql = + configuration + .applicationConfiguration() + .getOptional(ApplicationOptions.SQL) + .orElse(null); + if (sql == null) { return configuration; } try { - String value = DeflaterUtils.unzipString(flinkSql); - return configuration.withParameter( - configuration.parameter.mergeWith( - ParameterTool.fromMap( - Map.of(ConfigKeys.KEY_FLINK_SQL(), value)))); - } catch (Exception ignored) { - File sqlFile = new File(flinkSql); - try { - Map value = - PropertiesUtils.fromYamlFile(sqlFile.getAbsolutePath()); - return configuration.withParameter( - configuration.parameter.mergeWith(ParameterTool.fromMap(value))); - } catch (Exception e) { - throw new IllegalArgumentException("[StreamPark] init sql error." + e); + String decoded = DeflaterUtils.unzipString(sql); + Configuration updated = + Configuration.builder(configuration.applicationConfiguration()) + .set(ApplicationOptions.SQL, decoded, "decoded SQL argument") + .build(); + return configuration.withApplicationConfiguration(updated); + } catch (RuntimeException ignored) { + File sqlFile = new File(sql); + if (!sqlFile.isFile()) { + throw new ConfigException("SQL argument is neither compressed SQL nor a file: " + sql); } - } - } - - /** Table initialization result. */ - public static final class TableInitResult { - - public final ParameterTool parameter; - public final TableEnvironment tableEnv; - - public TableInitResult(ParameterTool parameter, TableEnvironment tableEnv) { - this.parameter = parameter; - this.tableEnv = tableEnv; - } - } - - /** Stream-table initialization result. */ - public static final class StreamTableInitResult { - - public final ParameterTool parameter; - public final StreamExecutionEnvironment streamEnv; - public final StreamTableEnvironment streamTableEnv; - - public StreamTableInitResult( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment streamTableEnv) { - this.parameter = parameter; - this.streamEnv = streamEnv; - this.streamTableEnv = streamTableEnv; + Configuration sqlConfiguration = ConfigurationParser.parse(Path.of(sql)); + Configuration updated = + Configuration.builder(configuration.applicationConfiguration()) + .add(sqlConfiguration) + .build(); + return configuration.withApplicationConfiguration(updated); } } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java deleted file mode 100644 index aa8d2152ef..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableTrait.java +++ /dev/null @@ -1,384 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.streampark.common.util.Utils; - -import org.apache.flink.api.common.JobExecutionResult; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.StatementSet; -import org.apache.flink.table.api.Table; -import org.apache.flink.table.api.TableConfig; -import org.apache.flink.table.api.TableDescriptor; -import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.TableException; -import org.apache.flink.table.api.TableResult; -import org.apache.flink.table.catalog.Catalog; -import org.apache.flink.table.catalog.CatalogDescriptor; -import org.apache.flink.table.expressions.Expression; -import org.apache.flink.table.functions.ScalarFunction; -import org.apache.flink.table.functions.UserDefinedFunction; -import org.apache.flink.table.module.Module; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; -import org.apache.flink.table.types.AbstractDataType; - -import java.util.List; -import java.util.Optional; - -/** Base table environment trait with SQL execution helpers. */ -public abstract class FlinkTableTrait implements TableEnvironment { - - public final ParameterTool parameter; - - private final TableEnvironment tableEnv; - - protected FlinkTableTrait(ParameterTool parameter, TableEnvironment tableEnv) { - this.parameter = parameter; - this.tableEnv = tableEnv; - } - - protected TableEnvironment getTableEnv() { - return tableEnv; - } - - public JobExecutionResult start() { - String appName = FlinkParameterUtils.getAppName(parameter, true); - return execute(appName); - } - - public JobExecutionResult execute(String jobName) { - Utils.printLogo("FlinkTable " + jobName + " Starting..."); - return null; - } - - public void sql(String sql) { - FlinkSqlExecutor.executeSql(sql, parameter, this); - } - - @Override - public Table fromValues(Expression... values) { - return tableEnv.fromValues(values); - } - - @Override - public Table fromValues(AbstractDataType rowType, Expression... values) { - return tableEnv.fromValues(rowType, values); - } - - @Override - public Table fromValues(Iterable values) { - return tableEnv.fromValues(values); - } - - @Override - public Table fromValues(AbstractDataType rowType, Iterable values) { - return tableEnv.fromValues(rowType, values); - } - - @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - tableEnv.createCatalog(catalogName, catalogDescriptor); - } - - @Override - public void useModules(String... moduleNames) { - tableEnv.useModules(moduleNames); - } - - @Override - public void createFunction( - String path, String className, List resourceUris) { - tableEnv.createFunction(path, className, resourceUris); - } - - @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - tableEnv.createFunction(path, className, resourceUris, ignoreIfExists); - } - - @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - tableEnv.createTemporaryFunction(path, className, resourceUris); - } - - @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - tableEnv.createTemporarySystemFunction(name, className, resourceUris); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - tableEnv.createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - tableEnv.createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor descriptor) { - return tableEnv.from(descriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return tableEnv.listFullModules(); - } - - @Override - public String[] listTables(String catalogName, String databaseName) { - return tableEnv.listTables(catalogName, databaseName); - } - - @Override - public String explainSql( - String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, format, extraDetails); - } - - @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return tableEnv.loadPlan(planReference); - } - - @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return tableEnv.compilePlanSql(statement); - } - - @Override - public void registerCatalog(String catalogName, Catalog catalog) { - tableEnv.registerCatalog(catalogName, catalog); - } - - @Override - public Optional getCatalog(String catalogName) { - return tableEnv.getCatalog(catalogName); - } - - @Override - public void loadModule(String moduleName, Module module) { - tableEnv.loadModule(moduleName, module); - } - - @Override - public void unloadModule(String moduleName) { - tableEnv.unloadModule(moduleName); - } - - @Override - public void createTemporarySystemFunction( - String name, Class functionClass) { - tableEnv.createTemporarySystemFunction(name, functionClass); - } - - @Override - public void createTemporarySystemFunction( - String name, UserDefinedFunction functionInstance) { - tableEnv.createTemporarySystemFunction(name, functionInstance); - } - - @Override - public boolean dropTemporarySystemFunction(String name) { - return tableEnv.dropTemporarySystemFunction(name); - } - - @Override - public void createFunction(String path, Class functionClass) { - tableEnv.createFunction(path, functionClass); - } - - @Override - public void createFunction( - String path, - Class functionClass, - boolean ignoreIfExists) { - tableEnv.createFunction(path, functionClass, ignoreIfExists); - } - - @Override - public boolean dropFunction(String path) { - return tableEnv.dropFunction(path); - } - - @Override - public void createTemporaryFunction( - String path, Class functionClass) { - tableEnv.createTemporaryFunction(path, functionClass); - } - - @Override - public void createTemporaryFunction(String path, UserDefinedFunction functionInstance) { - tableEnv.createTemporaryFunction(path, functionInstance); - } - - @Override - public boolean dropTemporaryFunction(String path) { - return tableEnv.dropTemporaryFunction(path); - } - - @Override - public void createTemporaryView(String path, Table view) { - tableEnv.createTemporaryView(path, view); - } - - @Override - public Table from(String path) { - return tableEnv.from(path); - } - - @Override - public String[] listCatalogs() { - return tableEnv.listCatalogs(); - } - - @Override - public String[] listModules() { - return tableEnv.listModules(); - } - - @Override - public String[] listDatabases() { - return tableEnv.listDatabases(); - } - - @Override - public String[] listTables() { - return tableEnv.listTables(); - } - - @Override - public String[] listViews() { - return tableEnv.listViews(); - } - - @Override - public String[] listTemporaryTables() { - return tableEnv.listTemporaryTables(); - } - - @Override - public String[] listTemporaryViews() { - return tableEnv.listTemporaryViews(); - } - - @Override - public String[] listUserDefinedFunctions() { - return tableEnv.listUserDefinedFunctions(); - } - - @Override - public String[] listFunctions() { - return tableEnv.listFunctions(); - } - - @Override - public boolean dropTemporaryTable(String path) { - return tableEnv.dropTemporaryTable(path); - } - - @Override - public boolean dropTemporaryView(String path) { - return tableEnv.dropTemporaryView(path); - } - - @Override - public String explainSql(String statement, ExplainDetail... extraDetails) { - return tableEnv.explainSql(statement, extraDetails); - } - - @Override - public Table sqlQuery(String query) { - return tableEnv.sqlQuery(query); - } - - @Override - public TableResult executeSql(String statement) { - return tableEnv.executeSql(statement); - } - - @Override - public String getCurrentCatalog() { - return tableEnv.getCurrentCatalog(); - } - - @Override - public void useCatalog(String catalogName) { - tableEnv.useCatalog(catalogName); - } - - @Override - public String getCurrentDatabase() { - return tableEnv.getCurrentDatabase(); - } - - @Override - public void useDatabase(String databaseName) { - tableEnv.useDatabase(databaseName); - } - - @Override - public TableConfig getConfig() { - return tableEnv.getConfig(); - } - - @Override - public StatementSet createStatementSet() { - return tableEnv.createStatementSet(); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void registerFunction(String name, ScalarFunction function) { - tableEnv.registerFunction(name, function); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void registerTable(String name, Table table) { - tableEnv.registerTable(name, table); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public Table scan(String... tablePath) { - return tableEnv.scan(tablePath); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public String[] getCompletionHints(String statement, int position) { - return tableEnv.getCompletionHints(statement, position); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommand.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommand.java index dd8ccad740..beba54cdcc 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommand.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommand.java @@ -17,8 +17,6 @@ package org.apache.streampark.flink.core; -import org.apache.commons.lang3.StringUtils; - import java.util.Optional; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -99,7 +97,7 @@ public enum SqlCommand { return Optional.empty(); } if (groups[0] == null) { - return Optional.of(new String[]{cleanUp(groups[0])}); + return Optional.of(new String[0]); } return Optional.of(new String[]{cleanUp(groups[1]), cleanUp(groups[2])}); }), @@ -125,8 +123,8 @@ public enum SqlCommand { private final String name; private final String regex; + private final Pattern pattern; private final SqlCommandConverter converter; - private Matcher matcher; SqlCommand(String name, String regex) { this(name, regex, SqlCommandConverters.DEFAULT); @@ -135,6 +133,7 @@ public enum SqlCommand { SqlCommand(String name, String regex, SqlCommandConverter converter) { this.name = name; this.regex = regex; + this.pattern = Pattern.compile(regex, PATTERN_FLAGS); this.converter = converter; } @@ -151,17 +150,21 @@ public SqlCommandConverter getConverter() { return converter; } - public Matcher getMatcher() { - return matcher; + public boolean matches(String input) { + return pattern.matcher(input).matches(); } - public boolean matches(String input) { - if (StringUtils.isBlank(regex)) { - return false; + Optional parseOperands(String input) { + Matcher matcher = pattern.matcher(input); + if (!matcher.matches()) { + return Optional.empty(); + } + + String[] groups = new String[matcher.groupCount()]; + for (int i = 0; i < groups.length; i++) { + groups[i] = matcher.group(i + 1); } - Pattern pattern = Pattern.compile(regex, PATTERN_FLAGS); - matcher = pattern.matcher(input); - return matcher.matches(); + return converter.convert(groups); } /** Resolve the first matching command for the given statement. */ diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandParser.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandParser.java index 15e991a2f5..40310b8b79 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandParser.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlCommandParser.java @@ -26,7 +26,6 @@ import java.util.List; import java.util.Optional; import java.util.function.Consumer; -import java.util.regex.Matcher; /** Parser for Flink SQL commands. */ public final class SqlCommandParser { @@ -100,26 +99,19 @@ public static List parseSQL(String sql, Consumer parseLine(SqlSegment sqlSegment) { - SqlCommand sqlCommand = SqlCommand.get(sqlSegment.sql.trim()); - if (sqlCommand == null) { - return Optional.empty(); - } - - Matcher matcher = sqlCommand.getMatcher(); - String[] groups = new String[matcher.groupCount()]; - for (int i = 0; i < groups.length; i++) { - groups[i] = matcher.group(i + 1); + String statement = sqlSegment.sql.trim(); + for (SqlCommand command : SqlCommand.values()) { + Optional operands = command.parseOperands(statement); + if (operands.isPresent()) { + return Optional.of( + new SqlCommandCall( + sqlSegment.start, + sqlSegment.end, + command, + operands.get(), + statement)); + } } - - return sqlCommand - .getConverter() - .convert(groups) - .map( - operands -> new SqlCommandCall( - sqlSegment.start, - sqlSegment.end, - sqlCommand, - operands, - sqlSegment.sql.trim())); + return Optional.empty(); } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlSplitter.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlSplitter.java index 1f148e74dd..a7f7665d84 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlSplitter.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/SqlSplitter.java @@ -17,7 +17,7 @@ package org.apache.streampark.flink.core; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.CommandLineParser; import org.apache.commons.lang3.StringUtils; @@ -38,7 +38,7 @@ public final class SqlSplitter { static { Set prefixes = new HashSet<>(); - prefixes.add(ConfigKeys.PARAM_PREFIX()); + prefixes.add(CommandLineParser.LONG_OPTION_PREFIX); SINGLE_LINE_COMMENT_PREFIX_LIST = Collections.unmodifiableSet(prefixes); } @@ -284,7 +284,8 @@ private static Map buildLineDescriptor(String sql) { lineNumber++; String line = scanner.nextLine().trim(); boolean nonEmpty = - StringUtils.isNotBlank(line) && !line.startsWith(ConfigKeys.PARAM_PREFIX()); + StringUtils.isNotBlank(line) + && !line.startsWith(CommandLineParser.LONG_OPTION_PREFIX); if (line.startsWith("/*")) { startComment = true; hasComment = true; @@ -333,7 +334,7 @@ private static String extractLineBreaks(String text) { } private static boolean isSingleLineComment(String text) { - return text.trim().startsWith(ConfigKeys.PARAM_PREFIX()); + return text.trim().startsWith(CommandLineParser.LONG_OPTION_PREFIX); } private static boolean isMultipleLineComment(String text) { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/StreamEnvConfig.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/StreamEnvConfig.java deleted file mode 100644 index 4812573ff9..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/StreamEnvConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -/** Stream execution environment configuration. */ -public class StreamEnvConfig { - - public final String[] args; - public final StreamEnvConfigFunction conf; - - public StreamEnvConfig(String[] args, StreamEnvConfigFunction conf) { - this.args = args; - this.conf = conf; - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/StreamTableEnvConfig.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/StreamTableEnvConfig.java deleted file mode 100644 index 670fc3df24..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/StreamTableEnvConfig.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -/** Stream and table environment configuration. */ -public class StreamTableEnvConfig { - - public final String[] args; - public final StreamEnvConfigFunction streamConfig; - public final TableEnvConfigFunction tableConfig; - - public StreamTableEnvConfig( - String[] args, - StreamEnvConfigFunction streamConfig, - TableEnvConfigFunction tableConfig) { - this.args = args; - this.streamConfig = streamConfig; - this.tableConfig = tableConfig; - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/TableEnvConfig.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/TableEnvConfig.java deleted file mode 100644 index a9d6636d2a..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/TableEnvConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -/** Table environment configuration. */ -public class TableEnvConfig { - - public final String[] args; - public final TableEnvConfigFunction conf; - - public TableEnvConfig(String[] args, TableEnvConfigFunction conf) { - this.args = args; - this.conf = conf; - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/TableEnvConfigFunction.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/TableEnvConfigFunction.java deleted file mode 100644 index a571194f98..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/TableEnvConfigFunction.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.table.api.TableConfig; - -@FunctionalInterface -public interface TableEnvConfigFunction { - - /** - * When used to initialize the TableEnvironment, it can be used to implement this function and - * customize the parameters to be set... - * - * @param tableConfig - * @param parameterTool - */ - void configuration(TableConfig tableConfig, ParameterTool parameterTool); -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/StreamEnvConfigFunction.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/bean/StreamContextSpec.java similarity index 65% rename from streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/StreamEnvConfigFunction.java rename to streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/bean/StreamContextSpec.java index d1ed71bb32..81a9534eaf 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/StreamEnvConfigFunction.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/bean/StreamContextSpec.java @@ -15,20 +15,19 @@ * limitations under the License. */ -package org.apache.streampark.flink.core; +package org.apache.streampark.flink.core.bean; + +import org.apache.streampark.flink.configuration.FlinkJobParameters; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.util.ParameterTool; -@FunctionalInterface -public interface StreamEnvConfigFunction { +public class StreamContextSpec { + + public final FlinkJobParameters parameter; + public final StreamExecutionEnvironment streamEnv; - /** - * When used to initialize StreamExecutionEnvironment, it can be used to implement this function - * and customize the parameters to be set... - * - * @param environment - * @param parameterTool - */ - void configuration(StreamExecutionEnvironment environment, ParameterTool parameterTool); + public StreamContextSpec(FlinkJobParameters parameter, StreamExecutionEnvironment streamEnv) { + this.parameter = parameter; + this.streamEnv = streamEnv; + } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/StreamEnvConfigFunction.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/bean/StreamTableContextSpec.java similarity index 53% rename from streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/StreamEnvConfigFunction.java rename to streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/bean/StreamTableContextSpec.java index 5c7ad54177..5019532a85 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/StreamEnvConfigFunction.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/bean/StreamTableContextSpec.java @@ -15,20 +15,25 @@ * limitations under the License. */ -package org.apache.streampark.flink.core; +package org.apache.streampark.flink.core.bean; + +import org.apache.streampark.flink.configuration.FlinkJobParameters; -import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; + +public class StreamTableContextSpec { -@FunctionalInterface -public interface StreamEnvConfigFunction { + public final FlinkJobParameters parameter; + public final StreamExecutionEnvironment streamEnv; + public final StreamTableEnvironment streamTableEnv; - /** - * When used to initialize StreamExecutionEnvironment, it can be used to implement this function - * and customize the parameters to be set... - * - * @param environment - * @param parameterTool - */ - void configuration(StreamExecutionEnvironment environment, ParameterTool parameterTool); + public StreamTableContextSpec( + FlinkJobParameters parameter, + StreamExecutionEnvironment streamEnv, + StreamTableEnvironment streamTableEnv) { + this.parameter = parameter; + this.streamEnv = streamEnv; + this.streamTableEnv = streamTableEnv; + } } diff --git a/streampark-flink/streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/HashCode.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/bean/TableContextSpec.java similarity index 64% rename from streampark-flink/streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/HashCode.java rename to streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/bean/TableContextSpec.java index 6399255ce5..c7df64ba04 100644 --- a/streampark-flink/streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/HashCode.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/bean/TableContextSpec.java @@ -15,25 +15,19 @@ * limitations under the License. */ -package org.apache.streampark.flink.udf; +package org.apache.streampark.flink.core.bean; -import org.apache.flink.table.functions.ScalarFunction; +import org.apache.streampark.flink.configuration.FlinkJobParameters; -public class HashCode extends ScalarFunction { +import org.apache.flink.table.api.TableEnvironment; - private Integer factor = 12; +public class TableContextSpec { - public HashCode() { - } - - public HashCode(Integer factor) { - this.factor = factor; - } + public final FlinkJobParameters parameter; + public final TableEnvironment tableEnv; - public Integer eval(String s) { - if (s == null) { - return 0; - } - return s.hashCode() * factor; + public TableContextSpec(FlinkJobParameters parameter, TableEnvironment tableEnv) { + this.parameter = parameter; + this.tableEnv = tableEnv; } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/FlinkConfiguration.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/FlinkConfiguration.java deleted file mode 100644 index 5bcfc40de4..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/FlinkConfiguration.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core.conf; - -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.configuration.Configuration; - -/** Flink runtime configuration holder. */ -public class FlinkConfiguration { - - public final ParameterTool parameter; - public final Configuration envConfig; - public final Configuration tableConfig; - - public FlinkConfiguration( - ParameterTool parameter, Configuration envConfig, Configuration tableConfig) { - this.parameter = parameter; - this.envConfig = envConfig; - this.tableConfig = tableConfig; - } - - public FlinkConfiguration withParameter(ParameterTool parameter) { - return new FlinkConfiguration(parameter, envConfig, tableConfig); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/ParameterCli.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/ParameterCli.java deleted file mode 100644 index 5a944d8612..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/conf/ParameterCli.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core.conf; - -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.util.PropertiesUtils; - -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.Options; - -import java.io.PrintStream; -import java.net.URLClassLoader; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** Parses Flink application CLI parameters from configuration files. */ -public final class ParameterCli { - - private static final String PROPERTY_PREFIX = ConfigKeys.KEY_FLINK_PROPERTY_PREFIX(); - private static final String OPTION_PREFIX = ConfigKeys.KEY_FLINK_OPTION_PREFIX(); - private static final String OPTION_MAIN = PROPERTY_PREFIX + "$internal.application.main"; - - private static final Options FLINK_OPTIONS = FlinkRunOption.allOptions(); - private static final DefaultParser PARSER = new DefaultParser(); - - private ParameterCli() { - } - - public static void emit(String output, PrintStream out) { - out.print(output); - } - - public static String read(String[] args) { - if ("--vmopt".equals(args[0])) { - return readVmOpt(); - } - return readConfigAction(args[0], args[1], Arrays.copyOfRange(args, 2, args.length)); - } - - private static String readVmOpt() { - ClassLoader classLoader = ClassLoader.getSystemClassLoader(); - if (classLoader instanceof URLClassLoader) { - return ""; - } - return "--add-opens java.base/jdk.internal.loader=ALL-UNNAMED " - + "--add-opens jdk.zipfs/jdk.nio.zipfs=ALL-UNNAMED"; - } - - private static String readConfigAction(String action, String conf, String[] programArgs) { - Map map = loadConfigMap(conf); - switch (action) { - case "--option": - return buildOptionString(map, programArgs); - case "--property": - return buildPropertyString(map); - case "--name": - return readAppName(map); - case "--detached": - return readDetachedMode(map, programArgs); - default: - return null; - } - } - - private static Map loadConfigMap(String conf) { - try { - String extension = conf.substring(conf.lastIndexOf('.') + 1).toLowerCase(); - switch (extension) { - case "yml": - case "yaml": - return PropertiesUtils.fromYamlFile(conf); - case "conf": - return PropertiesUtils.fromHoconFile(conf); - case "properties": - return PropertiesUtils.fromPropertiesFile(conf); - default: - throw new IllegalArgumentException( - "[StreamPark] Usage:flink.conf file error,must be (yml|conf|properties)"); - } - } catch (Exception e) { - return Collections.emptyMap(); - } - } - - private static String buildOptionString(Map map, String[] programArgs) { - String[] option = getOption(map, programArgs); - StringBuilder buffer = new StringBuilder(); - try { - org.apache.commons.cli.CommandLine line = PARSER.parse(FLINK_OPTIONS, option, false); - for (org.apache.commons.cli.Option x : line.getOptions()) { - buffer.append(" -").append(x.getOpt()); - if (x.hasArg()) { - buffer.append(" ").append(x.getValue()); - } - } - } catch (Exception exception) { - // Ignore invalid CLI options and continue with parsed values. - } - String mainClass = map.get(OPTION_MAIN); - if (mainClass != null) { - buffer.append(" -c ").append(mainClass); - } - return buffer.toString().trim(); - } - - private static String buildPropertyString(Map map) { - StringBuilder propertyBuffer = new StringBuilder(); - for (Map.Entry entry : map.entrySet()) { - appendPropertyEntry(propertyBuffer, entry.getKey(), entry.getValue()); - } - return propertyBuffer.toString().trim(); - } - - private static void appendPropertyEntry(StringBuilder propertyBuffer, String key, String value) { - if (OPTION_MAIN.equals(key) - || !key.startsWith(PROPERTY_PREFIX) - || value == null - || value.isEmpty()) { - return; - } - String propertyKey = key.substring(PROPERTY_PREFIX.length()).trim(); - String propertyValue = value.trim(); - propertyBuffer.append(" -D").append(propertyKey).append("="); - if (ConfigKeys.KEY_FLINK_APP_NAME().equals(propertyKey)) { - propertyBuffer.append(propertyValue.replace(" ", "_")); - } else { - propertyBuffer.append(propertyValue); - } - } - - private static String readAppName(Map map) { - String appName = - map.getOrDefault(PROPERTY_PREFIX.concat(ConfigKeys.KEY_FLINK_APP_NAME()), ""); - appName = appName.trim(); - return appName.isEmpty() ? "" : appName; - } - - private static String readDetachedMode(Map map, String[] programArgs) { - String[] detachedOption = getOption(map, programArgs); - try { - org.apache.commons.cli.CommandLine line = - PARSER.parse(FlinkRunOption.allOptions(), detachedOption, false); - boolean detached = - line.hasOption(FlinkRunOption.DETACHED_OPTION.getOpt()) - || line.hasOption(FlinkRunOption.DETACHED_OPTION.getLongOpt()); - return detached ? "Detached" : "Attach"; - } catch (Exception e) { - return "Attach"; - } - } - - public static String[] getOption(Map map, String[] args) { - Map optionMap = collectConfiguredOptions(map); - mergeProgramArgs(optionMap, args); - return toOptionArray(optionMap); - } - - private static Map collectConfiguredOptions(Map map) { - Map optionMap = new HashMap<>(); - for (Map.Entry entry : map.entrySet()) { - putConfiguredOption(optionMap, entry.getKey(), entry.getValue()); - } - return optionMap; - } - - private static void putConfiguredOption(Map optionMap, String key, String value) { - if (!key.startsWith(OPTION_PREFIX) || value == null || value.isEmpty()) { - return; - } - String optionKey = key.substring(OPTION_PREFIX.length()); - if (!FLINK_OPTIONS.hasOption(optionKey)) { - return; - } - if ("true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value)) { - if (Boolean.parseBoolean(value)) { - optionMap.put("-" + optionKey.trim(), true); - } - return; - } - optionMap.put("-" + optionKey.trim(), value); - } - - private static void mergeProgramArgs(Map optionMap, String[] args) { - if (args.length == 0) { - return; - } - try { - org.apache.commons.cli.CommandLine line = PARSER.parse(FLINK_OPTIONS, args, false); - for (org.apache.commons.cli.Option x : line.getOptions()) { - if (x.hasArg()) { - optionMap.put("-" + x.getLongOpt().trim(), x.getValue()); - } else { - optionMap.put("-" + x.getLongOpt().trim(), true); - } - } - } catch (Exception e) { - // Ignore invalid CLI options merged from program args. - } - } - - private static String[] toOptionArray(Map optionMap) { - List array = new ArrayList<>(); - for (Map.Entry entry : optionMap.entrySet()) { - array.add(entry.getKey()); - if (entry.getValue() instanceof String) { - array.add(entry.getValue().toString()); - } - } - return array.toArray(new String[0]); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkParameterUtils.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/util/FlinkParameterUtils.java similarity index 76% rename from streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkParameterUtils.java rename to streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/util/FlinkParameterUtils.java index f5c53e6926..e1b3f4d992 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base-v2/src/main/java/org/apache/streampark/flink/core/FlinkParameterUtils.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/util/FlinkParameterUtils.java @@ -15,16 +15,17 @@ * limitations under the License. */ -package org.apache.streampark.flink.core; +package org.apache.streampark.flink.util; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.option.ApplicationOptions; import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.DeflaterUtils; +import org.apache.streampark.flink.configuration.FlinkJobParameters; import org.apache.flink.configuration.PipelineOptions; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.util.ParameterTool; import javax.annotation.Nullable; @@ -34,20 +35,21 @@ public final class FlinkParameterUtils { private FlinkParameterUtils() { } - public static String getAppName(ParameterTool parameterTool) { + public static String getAppName(FlinkJobParameters parameterTool) { return getAppName(parameterTool, null, false); } - public static String getAppName(ParameterTool parameterTool, boolean required) { + public static String getAppName(FlinkJobParameters parameterTool, boolean required) { return getAppName(parameterTool, null, required); } public static String getAppName( - ParameterTool parameterTool, @Nullable String name, boolean required) { + FlinkJobParameters parameterTool, @Nullable String name, boolean required) { String appName; if (name == null) { appName = null; - String zippedAppName = parameterTool.get(ConfigKeys.KEY_APP_NAME(), null); + String zippedAppName = + parameterTool.getOptional(ApplicationOptions.NAME).orElse(null); if (zippedAppName != null) { try { appName = DeflaterUtils.unzipString(zippedAppName); @@ -56,7 +58,7 @@ public static String getAppName( } } if (appName == null) { - appName = parameterTool.get(ConfigKeys.KEY_FLINK_APP_NAME(), null); + appName = parameterTool.getOptional(FlinkOptions.PIPELINE_NAME).orElse(null); } } else { appName = name; @@ -68,7 +70,7 @@ public static String getAppName( return appName; } - public static T setAppName(T env, ParameterTool parameter) { + public static T setAppName(T env, FlinkJobParameters parameter) { String appName = getAppName(parameter); if (appName != null) { env.getConfig().getConfiguration().setString(PipelineOptions.NAME.key(), appName); @@ -77,7 +79,7 @@ public static T setAppName(T env, ParameterTool par } public static StreamTableEnvironment setAppName( - StreamTableEnvironment env, ParameterTool parameter) { + StreamTableEnvironment env, FlinkJobParameters parameter) { String appName = getAppName(parameter); if (appName != null) { env.getConfig().getConfiguration().setString(PipelineOptions.NAME.key(), appName); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/util/FlinkUtils.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/util/FlinkUtils.java index 6bf7d9ae36..ea4901d2e6 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/util/FlinkUtils.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/util/FlinkUtils.java @@ -21,7 +21,6 @@ import org.apache.flink.api.common.state.ListStateDescriptor; import org.apache.flink.api.common.typeinfo.TypeInformation; import org.apache.flink.runtime.state.FunctionInitializationContext; -import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions; import org.apache.flink.util.TimeUtils; import java.io.File; @@ -33,6 +32,8 @@ /** Flink utility methods. */ public final class FlinkUtils { + private static final String CHECKPOINT_INTERVAL_KEY = "execution.checkpointing.interval"; + private FlinkUtils() { } @@ -72,8 +73,7 @@ public static String getFlinkDistJar(String flinkHome) { public static boolean isCheckpointEnabled(Map map) { Duration checkpointInterval = TimeUtils.parseDuration( - map.getOrDefault( - ExecutionCheckpointingOptions.CHECKPOINTING_INTERVAL.key(), "0ms")); + map.getOrDefault(CHECKPOINT_INTERVAL_KEY, "0ms")); return checkpointInterval.toMillis() > 0; } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/ExitSecurityManager.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/configuration/FlinkJobParametersTest.java similarity index 51% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/ExitSecurityManager.java rename to streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/configuration/FlinkJobParametersTest.java index c50bbfb8d4..5c92832911 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/ExitSecurityManager.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/configuration/FlinkJobParametersTest.java @@ -15,28 +15,26 @@ * limitations under the License. */ -package org.apache.streampark.flink.client; +package org.apache.streampark.flink.configuration; -import java.security.Permission; +import org.apache.streampark.common.configuration.CommandLineParser; -/** Used to mask JVM requests for external operations. */ -public final class ExitSecurityManager extends SecurityManager { +import org.junit.jupiter.api.Test; - @Override - public void checkExit(int status) { - throw new SecurityException( - "System.exit(" - + status - + ") was called in your flink job, The job has been stopped, please check your program..."); - } +import static org.assertj.core.api.Assertions.assertThat; - @Override - public void checkPermission(Permission perm) { - // no-op - } +class FlinkJobParametersTest { + + @Test + void delegateToImmutableConfiguration() { + FlinkJobParameters parameters = + FlinkJobParameters.of( + CommandLineParser.parse( + new String[]{"--name", "demo", "--parallelism=-1", "--detached"})); - @Override - public void checkPermission(Permission perm, Object context) { - // no-op + assertThat(parameters.get("name")).isEqualTo("demo"); + assertThat(parameters.get("parallelism")).isEqualTo("-1"); + assertThat(parameters.get("detached")).isEqualTo("true"); + assertThat(parameters.toMap()).containsEntry("name", "demo"); } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/FlinkSqlExecutorTest.java similarity index 57% rename from streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableContext.java rename to streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/FlinkSqlExecutorTest.java index 4220885119..fd6aaa23bf 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/FlinkSqlExecutorTest.java @@ -17,21 +17,24 @@ package org.apache.streampark.flink.core; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.types.DataType; -/** Flink 1.17 table environment context. */ -public class TableContext extends FlinkTableTrait { +import org.junit.jupiter.api.Test; - public TableContext(ParameterTool parameter, TableEnvironment tableEnv) { - super(parameter, tableEnv); - } +import static org.assertj.core.api.Assertions.assertThat; - public TableContext(FlinkTableInitializer.TableInitResult init) { - this(init.parameter, init.tableEnv); - } +class FlinkSqlExecutorTest { + + @Test + void formatResolvedSchema() { + ResolvedSchema schema = + ResolvedSchema.physical( + new String[]{"id", "name"}, + new DataType[]{DataTypes.BIGINT(), DataTypes.STRING()}); - public TableContext(TableEnvConfig config) { - this(FlinkTableInitializer.initialize(config)); + assertThat(FlinkSqlExecutor.formatSchema(schema)) + .isEqualTo("Column\tType\nid\tBIGINT\nname\tSTRING\n"); } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/SqlCommandParserTest.java similarity index 50% rename from streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java rename to streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/SqlCommandParserTest.java index 23de2beaea..707face355 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/test/java/org/apache/streampark/flink/core/SqlCommandParserTest.java @@ -17,21 +17,31 @@ package org.apache.streampark.flink.core; -import org.apache.flink.kubernetes.kubeclient.FlinkKubeClient; -import org.apache.flink.kubernetes.kubeclient.decorators.ExternalServiceDecorator; -import org.apache.flink.kubernetes.kubeclient.resources.KubernetesService; +import org.junit.jupiter.api.Test; -import java.util.Optional; +import java.util.stream.IntStream; -/** Flink 1.17 Kubernetes client. */ -public class FlinkKubernetesClient extends FlinkKubernetesClientTrait { +import static org.assertj.core.api.Assertions.assertThat; - public FlinkKubernetesClient(FlinkKubeClient kubeClient) { - super(kubeClient); +class SqlCommandParserTest { + + @Test + void parseCommandOperandsWithoutSharedState() { + IntStream.range(0, 1_000) + .parallel() + .forEach( + index -> { + String key = "key-" + index; + String value = "value-" + index; + assertThat(SqlCommand.SET.parseOperands("SET " + key + "=" + value)) + .hasValueSatisfying( + operands -> assertThat(operands).containsExactly(key, value)); + }); } - @Override - public Optional getService(String serviceName) { - return kubeClient.getService(ExternalServiceDecorator.getExternalServiceName(serviceName)); + @Test + void parseSetWithoutAssignment() { + assertThat(SqlCommand.SET.parseOperands("SET")) + .hasValueSatisfying(operands -> assertThat(operands).isEmpty()); } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/pom.xml deleted file mode 100644 index cf6f2be43a..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/pom.xml +++ /dev/null @@ -1,100 +0,0 @@ - - - - 4.0.0 - - - org.apache.streampark - streampark-flink-shims - ${revision} - - - streampark-flink-shims-test - StreamPark : Flink Shims Test - - - - 1.17 - 1.17.0 - - - - - org.apache.streampark - streampark-common - ${project.version} - - - - org.apache.streampark - streampark-flink-shims-base - ${project.version} - - - - org.apache.streampark - streampark-flink-shims_flink-${streampark.flink.shims.version} - ${project.version} - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - test - - - - org.apache.flink - flink-streaming-java - ${flink.version} - test - - - - org.apache.flink - flink-clients - ${flink.version} - test - - - - org.apache.flink - flink-table-api-java-bridge - ${flink.version} - test - - - - org.junit.jupiter - junit-jupiter-engine - test - - - - - - apache-release - - true - - - - - diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FetchMillisecond.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FetchMillisecond.java deleted file mode 100644 index 67c38b14a7..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FetchMillisecond.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core.test; - -import org.apache.flink.table.functions.ScalarFunction; - -public class FetchMillisecond extends ScalarFunction { - - /** - * Get current milliseconds. - * - * @return Current milliseconds on the system. - */ - public Long eval() { - return System.currentTimeMillis(); - } - - /** - * Whether is a determined value.
- * return true: It represents that the function will be executed once at the stage of flink - * planner, then it will send the executed result back to runtime, it's a determined - * value.
- * return false: It represents that the function will be executed when the engine processed a row - * in the running stage. It's not a determined value.
- * Note: The function will return true by default, so the function need not be - * overwrite in general. - * - * @return true if it's a determined value else false. - */ - @Override - public boolean isDeterministic() { - return false; - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlExecuteTest.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlExecuteTest.java deleted file mode 100644 index b718de2391..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlExecuteTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core.test; - -import org.apache.streampark.common.conf.ConfigKeys; -import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.flink.core.FlinkSqlExecutor; -import org.apache.streampark.flink.core.FlinkTableInitializer; -import org.apache.streampark.flink.core.StreamTableContext; - -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; - -class FlinkSqlExecuteTest { - - private void execute(String sql) { - List args = new ArrayList<>(); - args.add(ConfigKeys.KEY_FLINK_SQL(ConfigKeys.PARAM_PREFIX())); - args.add(DeflaterUtils.zipString(sql)); - args.add(ConfigKeys.KEY_FLINK_CONF(ConfigKeys.PARAM_PREFIX())); - args.add(DeflaterUtils.zipString("execution.runtime-mode: streaming")); - StreamTableContext context = - new StreamTableContext(FlinkTableInitializer.initialize(args.toArray(new String[0]), null, null)); - FlinkSqlExecutor.executeSql(ConfigKeys.KEY_FLINK_SQL(), context.parameter, context); - } - - @Test - void execute() { - execute( - "-- set -------\n" - + "set 'table.local-time-zone' = 'GMT+08:00';\n" - + "\n" - + "-- reset -----\n" - + "reset 'table.local-time-zone';\n" - + "reset;\n" - + "\n" - + "CREATE temporary TABLE source_kafka1(\n" - + " `id` int,\n" - + " `name` string,\n" - + " `age` int\n" - + ") WITH (\n" - + " 'connector' = 'datagen',\n" - + " 'rows-per-second' = '1',\n" - + " 'number-of-rows' = '1'\n" - + ");\n" - + "\n" - + "create table sink_kafka1(\n" - + " `id` int,\n" - + " `name` string,\n" - + " `age` int\n" - + ") with (\n" - + " 'connector' = 'print'\n" - + ");\n" - + "\n" - + "insert into sink_kafka1\n" - + "select id, name, age\n" - + "from source_kafka1;\n" - + "\n"); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlValidationTest.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlValidationTest.java deleted file mode 100644 index 574a11a206..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-test/src/test/java/org/apache/streampark/flink/core/test/FlinkSqlValidationTest.java +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core.test; - -import org.apache.streampark.flink.core.FlinkSqlValidationResult; -import org.apache.streampark.flink.core.FlinkSqlValidator; - -import org.junit.jupiter.api.Test; - -import java.util.function.Consumer; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -class FlinkSqlValidationTest { - - private void verify(String sql, Consumer func) { - String insert = - "CREATE TABLE my_source ( `f` BIGINT ) WITH ( 'connector' = 'datagen' );\n" - + "create TABLE my_sink(`f` BIGINT ) with ('connector' = 'print');\n" - + "insert into my_sink select * from my_source;\n"; - - func.accept(FlinkSqlValidator.verifySql(sql + "\n" + insert)); - } - - @Test - void select() { - verify( - "-- select ---\n" - + "select id, name, age, fetch_millisecond() as millisecond\n" - + "from source_kafka1;\n", - r -> assertTrue(r.success())); - } - - @Test - void setReset() { - verify( - "-- set -------\n" - + "set 'table.local-time-zone' = 'GMT+08:00';\n" - + "\n" - + "-- reset -----\n" - + "reset 'table.local-time-zone';\n" - + "reset;\n", - r -> assertTrue(r.success())); - } - - @Test - void create() { - verify( - "-- create --\n" - + "CREATE TABLE Orders (\n" - + " `user` BIGINT,\n" - + " product STRING,\n" - + " order_time TIMESTAMP(3)\n" - + ") WITH (\n" - + " 'connector' = 'kafka',\n" - + " 'scan.startup.mode' = 'earliest-offset'\n" - + ");\n" - + "\n" - + "create catalog hive with (\n" - + " 'type' = 'hive',\n" - + " 'hadoop-conf-dir' = '$path',\n" - + " 'hive-conf-dir' = '$path'\n" - + ");\n" - + "create database test;\n" - + "\n" - + "create view if not exists a.b.test_view123 as\n" - + "select id, name, age, fetch_millisecond() as millisecond\n" - + "from source_kafka1;\n" - + "\n" - + "create function fetch_millisecond as " - + "'org.apache.streampark.flink.core.test.FetchMillisecond' language java;\n" - + "create temporary function fetch_millisecond as " - + "'org.apache.streampark.flink.core.test.FetchMillisecond' language java;\n" - + "create TEMPORARY SYSTEM function fetch_millisecond as " - + "'org.apache.streampark.flink.core.test.FetchMillisecond' language java;\n" - + "\n", - r -> assertTrue(r.success())); - } - - @Test - void drop() { - verify( - "drop catalog hive;\n" - + "drop catalog IF EXISTS hive;\n" - + "\n" - + "drop table test;\n" - + "drop TEMPORARY table test;\n" - + "drop table IF EXISTS test;\n" - + "drop TEMPORARY table IF EXISTS test;\n" - + "drop TEMPORARY table IF EXISTS test.test;\n" - + "drop TEMPORARY table IF EXISTS hive.test.test;\n" - + "\n" - + "drop database test;\n" - + "drop database if exists test;\n" - + "drop database if exists hive.test;\n" - + "drop database if exists hive.test RESTRICT;\n" - + "drop database if exists hive.test CASCADE;\n" - + "\n" - + "drop view test_view;\n" - + "drop TEMPORARY view test_view;\n" - + "drop view IF EXISTS test_view;\n" - + "drop TEMPORARY view IF EXISTS test_view;\n" - + "drop view test.test_view;\n" - + "\n" - + "drop function test_function;\n" - + "drop TEMPORARY function test_function;\n" - + "drop TEMPORARY SYSTEM function test_function;\n" - + "drop function IF EXISTS test_function;\n" - + "\n", - r -> assertTrue(r.success())); - } - - @Test - void alter() { - verify( - "alter table test rename to test1;\n" - + "alter table test set ('k' = 'v');\n" - + "\n" - + "alter view test_view rename to test_view2;\n" - + "alter view test_view as\n" - + "select id, name, age, fetch_millisecond() as millisecond\n" - + "from source_kafka1;\n" - + "\n" - + "alter temporary function fetch_millisecond as " - + "'org.apache.streampark.flink.core.test.FetchMillisecond' language java;\n" - + "alter TEMPORARY SYSTEM function fetch_millisecond as " - + "'org.apache.streampark.flink.core.test.FetchMillisecond' language java;\n" - + "\n", - r -> assertTrue(r.success())); - } - - @Test - void desc() { - verify( - "-- desc ---\n" - + "DESCRIBE test;\n" - + "desc test;\n" - + "desc test.test;\n", - r -> assertTrue(r.success())); - } - - @Test - void explain() { - verify( - "-- explain ---\n" - + "EXPLAIN PLAN FOR\n" - + "select id, name, age, fetch_millisecond() as millisecond\n" - + "from source_kafka1;\n" - + "\n" - + "EXPLAIN ESTIMATED_COST\n" - + "select id, name, age, fetch_millisecond() as millisecond\n" - + "from source_kafka1;\n", - r -> assertTrue(r.success())); - } - - @Test - void use() { - verify( - "-- use ---\n" - + "use catalog hive;\n" - + "\n" - + "use modules hive;\n" - + "use modules hive, core;\n" - + "\n" - + "use test;\n" - + "use hive.test;\n" - + "\n", - r -> assertTrue(r.success())); - } - - @Test - void show() { - verify( - "-- show ---\n" - + "show catalogs;\n" - + "\n" - + "show current catalog;\n" - + "\n" - + "show databases;\n" - + "\n" - + "show current database;\n" - + "\n" - + "show tables;\n" - + "show tables from db1;\n" - + "show tables from db1 like '%n';\n" - + "show tables from db1 not like '%n';\n" - + "\n" - + "show create table test;\n" - + "\n" - + "show columns from orders;\n" - + "show columns from database1.orders;\n" - + "show columns from catalog1.database1.orders;\n" - + "show columns from orders like '%r';\n" - + "show columns from orders not like '%_r';\n" - + "\n" - + "show views;\n" - + "\n" - + "show create view test_view;\n" - + "\n" - + "show functions;\n" - + "show user functions;\n" - + "\n" - + "show modules;\n" - + "show full modules;\n" - + "\n", - r -> assertTrue(r.success())); - } - - @Test - void load() { - verify( - "-- load ---\n" - + "load module hive;\n" - + "load module hive with ('hive-version' = '2.3.6');\n" - + "\n" - + "-- unload ---\n" - + "unload module hive;\n" - + "\n", - r -> assertTrue(r.success())); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/pom.xml deleted file mode 100644 index 0ed2f685b0..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/pom.xml +++ /dev/null @@ -1,114 +0,0 @@ - - - - 4.0.0 - - - org.apache.streampark - streampark-flink-shims - ${revision} - - - streampark-flink-shims_flink-1.17 - StreamPark : Flink Shims 1.17 - - - 1.17.0 - - - - - org.apache.streampark - streampark-flink-shims-base - ${project.version} - - - - org.apache.flink - flink-table-planner_${scala.binary.version} - ${flink.version} - provided - - - - org.apache.flink - flink-streaming-java - ${flink.version} - provided - - - - org.apache.flink - flink-table-api-java - ${flink.version} - provided - - - - org.apache.flink - flink-table-api-java-bridge - ${flink.version} - provided - - - - org.apache.flink - flink-clients - ${flink.version} - provided - - - - org.apache.flink - flink-statebackend-rocksdb - ${flink.version} - provided - - - - org.apache.flink - flink-yarn - ${flink.version} - provided - - - - org.apache.hadoop - hadoop-client-api - true - - - - org.apache.hadoop - hadoop-client-runtime - true - - - - org.apache.flink - flink-kubernetes - ${flink.version} - provided - - - - - - - diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java deleted file mode 100644 index 395988cc59..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.api.common.JobID; -import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.core.execution.SavepointFormatType; - -import java.util.concurrent.CompletableFuture; - -/** Flink 1.17 cluster client with native/canonical savepoint format support. */ -public class FlinkClusterClient extends FlinkClientTrait { - - public FlinkClusterClient(ClusterClient clusterClient) { - super(clusterClient); - } - - @Override - public CompletableFuture triggerSavepoint( - JobID jobID, String savepointDir, boolean nativeFormat) { - return clusterClient.triggerSavepoint( - jobID, - savepointDir, - nativeFormat ? SavepointFormatType.NATIVE : SavepointFormatType.CANONICAL); - } - - @Override - public CompletableFuture cancelWithSavepoint( - JobID jobID, String savepointDirectory, boolean nativeFormat) { - return clusterClient.cancelWithSavepoint( - jobID, - savepointDirectory, - nativeFormat ? SavepointFormatType.NATIVE : SavepointFormatType.CANONICAL); - } - - @Override - public CompletableFuture stopWithSavepoint( - JobID jobID, - boolean advanceToEndOfEventTime, - String savepointDirectory, - boolean nativeFormat) { - return clusterClient.stopWithSavepoint( - jobID, - advanceToEndOfEventTime, - savepointDirectory, - nativeFormat ? SavepointFormatType.NATIVE : SavepointFormatType.CANONICAL); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java deleted file mode 100644 index 9369100da4..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.bridge.java.StreamStatementSet; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; - -/** Flink 1.17 stream-table environment context. */ -public class StreamTableContext extends FlinkStreamTableTrait { - - public StreamTableContext( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment tableEnv) { - super(parameter, streamEnv, tableEnv); - } - - public StreamTableContext(FlinkTableInitializer.StreamTableInitResult init) { - this(init.parameter, init.streamEnv, init.streamTableEnv); - } - - public StreamTableContext(StreamTableEnvConfig config) { - this(FlinkTableInitializer.initialize(config)); - } - - @Override - public StreamStatementSet createStatementSet() { - return getStreamTableEnv().createStatementSet(); - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableExt.java deleted file mode 100644 index a7181f9c21..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.17/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.types.Row; - -/** Table API extensions for Flink 1.17 Java stream-table applications. */ -public final class TableExt { - - private TableExt() { - } - - /** Table alias helper (Scala {@code ->} operator equivalent: {@code as}). */ - public static final class Table { - - private final org.apache.flink.table.api.Table flinkTable; - - public Table(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - public org.apache.flink.table.api.Table as(String field, String... fields) { - return flinkTable.as(field, fields); - } - } - - /** Table-to-DataStream conversion helpers. */ - public static class TableConversions { - - private final org.apache.flink.table.api.Table flinkTable; - - public TableConversions(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - /** Changelog stream conversion (Scala {@code \\} operator equivalent). */ - public DataStream toChangelogDataStream(StreamTableContext context) { - context.isConvertedToDataStream = true; - return context.toDataStream(flinkTable); - } - - /** Append stream conversion (Scala {@code >>} operator equivalent). */ - public DataStream toAppendDataStream( - TypeInformation typeInfo, StreamTableContext context) { - context.isConvertedToDataStream = true; - return context.toAppendStream(flinkTable, typeInfo); - } - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/pom.xml index 43ddddf24d..742d69a1dc 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/pom.xml +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/pom.xml @@ -107,8 +107,4 @@ provided - - - - diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java index 5aa86b628e..9d6baa69a3 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java @@ -24,7 +24,7 @@ import java.util.concurrent.CompletableFuture; /** Flink 1.18 cluster client with native/canonical savepoint format support. */ -public class FlinkClusterClient extends FlinkClientTrait { +public class FlinkClusterClient extends AbstractFlinkClient { public FlinkClusterClient(ClusterClient clusterClient) { super(clusterClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java index e449354546..41808e2986 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java @@ -23,7 +23,7 @@ import java.util.Optional; /** Flink 1.18 Kubernetes client. */ -public class FlinkKubernetesClient extends FlinkKubernetesClientTrait { +public class FlinkKubernetesClient extends AbstractFlinkKubernetesClient { public FlinkKubernetesClient(FlinkKubeClient kubeClient) { super(kubeClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index f4e14c64e8..52a48dd8e6 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -17,27 +17,47 @@ package org.apache.streampark.flink.core; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.streampark.flink.core.bean.StreamTableContextSpec; + +import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.table.api.bridge.java.StreamStatementSet; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.table.functions.AggregateFunction; +import org.apache.flink.table.functions.TableAggregateFunction; +import org.apache.flink.table.functions.TableFunction; -/** Flink 1.18 stream-table environment context. */ -public class StreamTableContext extends FlinkStreamTableTrait { +/** + * Flink 1.18 stream-table environment context. + * + *

Legacy registration methods remain in this version-specific adapter because Flink 2.x removed + * them from {@code StreamTableEnvironment}. Keeping them outside the shared base prevents 1.x API + * references from leaking into Flink 2.x shims. + */ +public class StreamTableContext extends AbstractFlinkStreamTable { - public StreamTableContext( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment tableEnv) { - super(parameter, streamEnv, tableEnv); + public StreamTableContext(StreamTableContextSpec streamContextConfig) { + super(streamContextConfig.parameter, streamContextConfig.streamEnv, streamContextConfig.streamTableEnv); } - public StreamTableContext(FlinkTableInitializer.StreamTableInitResult init) { - this(init.parameter, init.streamEnv, init.streamTableEnv); + @Override + public void registerFunction(String name, TableFunction tableFunction) { + getStreamTableEnv().registerFunction(name, tableFunction); } - public StreamTableContext(StreamTableEnvConfig config) { - this(FlinkTableInitializer.initialize(config)); + @Override + public void registerFunction(String name, AggregateFunction aggregateFunction) { + getStreamTableEnv().registerFunction(name, aggregateFunction); + } + + @Override + public void registerFunction( + String name, + TableAggregateFunction tableAggregateFunction) { + getStreamTableEnv().registerFunction(name, tableAggregateFunction); + } + + @Override + public void registerDataStream(String name, DataStream dataStream) { + getStreamTableEnv().registerDataStream(name, dataStream); } @Override diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableContext.java index eefa7a9d1b..f2b1868bc2 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -17,21 +17,25 @@ package org.apache.streampark.flink.core; -import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.core.bean.TableContextSpec; + import org.apache.flink.table.api.TableEnvironment; +import org.apache.flink.table.catalog.CatalogDescriptor; /** Flink 1.18 table environment context. */ -public class TableContext extends FlinkTableTrait { +public class TableContext extends AbstractFlinkTable { - public TableContext(ParameterTool parameter, TableEnvironment tableEnv) { + public TableContext(FlinkJobParameters parameter, TableEnvironment tableEnv) { super(parameter, tableEnv); } - public TableContext(FlinkTableInitializer.TableInitResult init) { - this(init.parameter, init.tableEnv); + public TableContext(TableContextSpec contextConfig) { + this(contextConfig.parameter, contextConfig.tableEnv); } - public TableContext(TableEnvConfig config) { - this(FlinkTableInitializer.initialize(config)); + @Override + public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { + getTableEnv().createCatalog(catalogName, catalogDescriptor); } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableExt.java deleted file mode 100644 index e1c0d00f20..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.18/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.types.Row; - -/** Table API extensions for Flink 1.18 Java stream-table applications. */ -public final class TableExt { - - private TableExt() { - } - - /** Table alias helper (Scala {@code ->} operator equivalent: {@code as}). */ - public static final class Table { - - private final org.apache.flink.table.api.Table flinkTable; - - public Table(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - public org.apache.flink.table.api.Table as(String field, String... fields) { - return flinkTable.as(field, fields); - } - } - - /** Table-to-DataStream conversion helpers. */ - public static class TableConversions { - - private final org.apache.flink.table.api.Table flinkTable; - - public TableConversions(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - /** Changelog stream conversion (Scala {@code \\} operator equivalent). */ - public DataStream toChangelogDataStream(StreamTableContext context) { - context.isConvertedToDataStream = true; - return context.toDataStream(flinkTable); - } - - /** Append stream conversion (Scala {@code >>} operator equivalent). */ - public DataStream toAppendDataStream( - TypeInformation typeInfo, StreamTableContext context) { - context.isConvertedToDataStream = true; - return context.toAppendStream(flinkTable, typeInfo); - } - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/pom.xml index c1c28a8432..788a0d9bef 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/pom.xml +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/pom.xml @@ -108,7 +108,4 @@ - - - diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java index d6e94b8200..b196054731 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java @@ -24,15 +24,15 @@ import java.util.concurrent.CompletableFuture; /** Flink 1.19 cluster client with native/canonical savepoint format support. */ -public class FlinkClusterClient extends FlinkClientTrait { +public class FlinkClusterClient extends AbstractFlinkClient { public FlinkClusterClient(ClusterClient clusterClient) { super(clusterClient); } @Override - public CompletableFuture triggerSavepoint( - JobID jobID, String savepointDir, boolean nativeFormat) { + public CompletableFuture triggerSavepoint(JobID jobID, + String savepointDir, boolean nativeFormat) { return clusterClient.triggerSavepoint( jobID, savepointDir, @@ -40,8 +40,9 @@ public CompletableFuture triggerSavepoint( } @Override - public CompletableFuture cancelWithSavepoint( - JobID jobID, String savepointDirectory, boolean nativeFormat) { + public CompletableFuture cancelWithSavepoint(JobID jobID, + String savepointDirectory, + boolean nativeFormat) { return clusterClient.cancelWithSavepoint( jobID, savepointDirectory, @@ -49,8 +50,7 @@ public CompletableFuture cancelWithSavepoint( } @Override - public CompletableFuture stopWithSavepoint( - JobID jobID, + public CompletableFuture stopWithSavepoint(JobID jobID, boolean advanceToEndOfEventTime, String savepointDirectory, boolean nativeFormat) { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java index f0da44ef0f..75073d2a1a 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java @@ -23,7 +23,7 @@ import java.util.Optional; /** Flink 1.19 Kubernetes client. */ -public class FlinkKubernetesClient extends FlinkKubernetesClientTrait { +public class FlinkKubernetesClient extends AbstractFlinkKubernetesClient { public FlinkKubernetesClient(FlinkKubeClient kubeClient) { super(kubeClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index 364bb543c8..012f9408be 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -17,31 +17,52 @@ package org.apache.streampark.flink.core; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.streampark.flink.core.bean.StreamTableContextSpec; + +import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.table.api.bridge.java.StreamStatementSet; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.table.functions.AggregateFunction; +import org.apache.flink.table.functions.TableAggregateFunction; +import org.apache.flink.table.functions.TableFunction; + +/** + * Flink 1.19 stream-table environment context. + * + *

Legacy registration methods remain in this version-specific adapter because Flink 2.x removed + * them from {@code StreamTableEnvironment}. Keeping them outside the shared base prevents 1.x API + * references from leaking into Flink 2.x shims. + */ +public class StreamTableContext extends AbstractFlinkStreamTable { + + public StreamTableContext(StreamTableContextSpec streamContextConfig) { + super(streamContextConfig.parameter, streamContextConfig.streamEnv, streamContextConfig.streamTableEnv); + } -/** Flink 1.19 stream-table environment context. */ -public class StreamTableContext extends FlinkStreamTableTrait { + @Override + public void registerFunction(String name, TableFunction tableFunction) { + getStreamTableEnv().registerFunction(name, tableFunction); + } - public StreamTableContext( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment tableEnv) { - super(parameter, streamEnv, tableEnv); + @Override + public void registerFunction(String name, AggregateFunction aggregateFunction) { + getStreamTableEnv().registerFunction(name, aggregateFunction); } - public StreamTableContext(FlinkTableInitializer.StreamTableInitResult init) { - this(init.parameter, init.streamEnv, init.streamTableEnv); + @Override + public void registerFunction( + String name, + TableAggregateFunction tableAggregateFunction) { + getStreamTableEnv().registerFunction(name, tableAggregateFunction); } - public StreamTableContext(StreamTableEnvConfig config) { - this(FlinkTableInitializer.initialize(config)); + @Override + public void registerDataStream(String name, DataStream dataStream) { + getStreamTableEnv().registerDataStream(name, dataStream); } @Override public StreamStatementSet createStatementSet() { return getStreamTableEnv().createStatementSet(); } + } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableContext.java index 59596b2164..90e7446b1d 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -17,21 +17,20 @@ package org.apache.streampark.flink.core; -import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.core.bean.TableContextSpec; + import org.apache.flink.table.api.TableEnvironment; /** Flink 1.19 table environment context. */ -public class TableContext extends FlinkTableTrait { +public class TableContext extends AbstractFlinkTable { - public TableContext(ParameterTool parameter, TableEnvironment tableEnv) { + public TableContext(FlinkJobParameters parameter, TableEnvironment tableEnv) { super(parameter, tableEnv); } - public TableContext(FlinkTableInitializer.TableInitResult init) { - this(init.parameter, init.tableEnv); + public TableContext(TableContextSpec contextConfig) { + this(contextConfig.parameter, contextConfig.tableEnv); } - public TableContext(TableEnvConfig config) { - this(FlinkTableInitializer.initialize(config)); - } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableExt.java deleted file mode 100644 index 25581ad462..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.19/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.types.Row; - -/** Table API extensions for Flink 1.19 Java stream-table applications. */ -public final class TableExt { - - private TableExt() { - } - - /** Table alias helper (Scala {@code ->} operator equivalent: {@code as}). */ - public static final class Table { - - private final org.apache.flink.table.api.Table flinkTable; - - public Table(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - public org.apache.flink.table.api.Table as(String field, String... fields) { - return flinkTable.as(field, fields); - } - } - - /** Table-to-DataStream conversion helpers. */ - public static class TableConversions { - - private final org.apache.flink.table.api.Table flinkTable; - - public TableConversions(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - /** Changelog stream conversion (Scala {@code \\} operator equivalent). */ - public DataStream toChangelogDataStream(StreamTableContext context) { - context.isConvertedToDataStream = true; - return context.toDataStream(flinkTable); - } - - /** Append stream conversion (Scala {@code >>} operator equivalent). */ - public DataStream toAppendDataStream( - TypeInformation typeInfo, StreamTableContext context) { - context.isConvertedToDataStream = true; - return context.toAppendStream(flinkTable, typeInfo); - } - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/pom.xml index 4a0318364e..e65c8e2149 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/pom.xml +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/pom.xml @@ -108,7 +108,4 @@ - - - diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java index 7ff543fbb0..66db8acb38 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java @@ -24,7 +24,7 @@ import java.util.concurrent.CompletableFuture; /** Flink 1.20 cluster client with native/canonical savepoint format support. */ -public class FlinkClusterClient extends FlinkClientTrait { +public class FlinkClusterClient extends AbstractFlinkClient { public FlinkClusterClient(ClusterClient clusterClient) { super(clusterClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java index 501432500d..ab3d618170 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java @@ -23,7 +23,7 @@ import java.util.Optional; /** Flink 1.20 Kubernetes client. */ -public class FlinkKubernetesClient extends FlinkKubernetesClientTrait { +public class FlinkKubernetesClient extends AbstractFlinkKubernetesClient { public FlinkKubernetesClient(FlinkKubeClient kubeClient) { super(kubeClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index b680dc4e6b..7d907c1968 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -17,31 +17,52 @@ package org.apache.streampark.flink.core; -import org.apache.flink.api.java.utils.ParameterTool; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.streampark.flink.core.bean.StreamTableContextSpec; + +import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.table.api.bridge.java.StreamStatementSet; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.table.functions.AggregateFunction; +import org.apache.flink.table.functions.TableAggregateFunction; +import org.apache.flink.table.functions.TableFunction; + +/** + * Flink 1.20 stream-table environment context. + * + *

Legacy registration methods remain in this version-specific adapter because Flink 2.x removed + * them from {@code StreamTableEnvironment}. Keeping them outside the shared base prevents 1.x API + * references from leaking into Flink 2.x shims. + */ +public class StreamTableContext extends AbstractFlinkStreamTable { + + public StreamTableContext(StreamTableContextSpec streamContextConfig) { + super(streamContextConfig.parameter, streamContextConfig.streamEnv, streamContextConfig.streamTableEnv); + } -/** Flink 1.20 stream-table environment context. */ -public class StreamTableContext extends FlinkStreamTableTrait { + @Override + public void registerFunction(String name, TableFunction tableFunction) { + getStreamTableEnv().registerFunction(name, tableFunction); + } - public StreamTableContext( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment tableEnv) { - super(parameter, streamEnv, tableEnv); + @Override + public void registerFunction(String name, AggregateFunction aggregateFunction) { + getStreamTableEnv().registerFunction(name, aggregateFunction); } - public StreamTableContext(FlinkTableInitializer.StreamTableInitResult init) { - this(init.parameter, init.streamEnv, init.streamTableEnv); + @Override + public void registerFunction( + String name, + TableAggregateFunction tableAggregateFunction) { + getStreamTableEnv().registerFunction(name, tableAggregateFunction); } - public StreamTableContext(StreamTableEnvConfig config) { - this(FlinkTableInitializer.initialize(config)); + @Override + public void registerDataStream(String name, DataStream dataStream) { + getStreamTableEnv().registerDataStream(name, dataStream); } @Override public StreamStatementSet createStatementSet() { return getStreamTableEnv().createStatementSet(); } + } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableContext.java index 397497860d..83b340302f 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -17,21 +17,20 @@ package org.apache.streampark.flink.core; -import org.apache.flink.api.java.utils.ParameterTool; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.core.bean.TableContextSpec; + import org.apache.flink.table.api.TableEnvironment; /** Flink 1.20 table environment context. */ -public class TableContext extends FlinkTableTrait { +public class TableContext extends AbstractFlinkTable { - public TableContext(ParameterTool parameter, TableEnvironment tableEnv) { + public TableContext(FlinkJobParameters parameter, TableEnvironment tableEnv) { super(parameter, tableEnv); } - public TableContext(FlinkTableInitializer.TableInitResult init) { - this(init.parameter, init.tableEnv); + public TableContext(TableContextSpec contextConfig) { + this(contextConfig.parameter, contextConfig.tableEnv); } - public TableContext(TableEnvConfig config) { - this(FlinkTableInitializer.initialize(config)); - } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableExt.java deleted file mode 100644 index 63cd519b33..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-1.20/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.types.Row; - -/** Table API extensions for Flink 1.20 Java stream-table applications. */ -public final class TableExt { - - private TableExt() { - } - - /** Table alias helper (Scala {@code ->} operator equivalent: {@code as}). */ - public static final class Table { - - private final org.apache.flink.table.api.Table flinkTable; - - public Table(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - public org.apache.flink.table.api.Table as(String field, String... fields) { - return flinkTable.as(field, fields); - } - } - - /** Table-to-DataStream conversion helpers. */ - public static class TableConversions { - - private final org.apache.flink.table.api.Table flinkTable; - - public TableConversions(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - /** Changelog stream conversion (Scala {@code \\} operator equivalent). */ - public DataStream toChangelogDataStream(StreamTableContext context) { - context.isConvertedToDataStream = true; - return context.toDataStream(flinkTable); - } - - /** Append stream conversion (Scala {@code >>} operator equivalent). */ - public DataStream toAppendDataStream( - TypeInformation typeInfo, StreamTableContext context) { - context.isConvertedToDataStream = true; - return context.toAppendStream(flinkTable, typeInfo); - } - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/pom.xml index 3575626581..6acf144d40 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/pom.xml +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/pom.xml @@ -35,7 +35,7 @@ org.apache.streampark - streampark-flink-shims-base-v2 + streampark-flink-shims-base ${project.version} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java index 62deeb412c..2d54208ea3 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java @@ -24,15 +24,15 @@ import java.util.concurrent.CompletableFuture; /** Flink 2.0 cluster client with native/canonical savepoint format support. */ -public class FlinkClusterClient extends FlinkClientTrait { +public class FlinkClusterClient extends AbstractFlinkClient { public FlinkClusterClient(ClusterClient clusterClient) { super(clusterClient); } @Override - public CompletableFuture triggerSavepoint( - JobID jobID, String savepointDir, boolean nativeFormat) { + public CompletableFuture triggerSavepoint(JobID jobID, String savepointDir, + boolean nativeFormat) { return clusterClient.triggerSavepoint( jobID, savepointDir, @@ -40,8 +40,8 @@ public CompletableFuture triggerSavepoint( } @Override - public CompletableFuture cancelWithSavepoint( - JobID jobID, String savepointDirectory, boolean nativeFormat) { + public CompletableFuture cancelWithSavepoint(JobID jobID, String savepointDirectory, + boolean nativeFormat) { return clusterClient.cancelWithSavepoint( jobID, savepointDirectory, @@ -49,8 +49,7 @@ public CompletableFuture cancelWithSavepoint( } @Override - public CompletableFuture stopWithSavepoint( - JobID jobID, + public CompletableFuture stopWithSavepoint(JobID jobID, boolean advanceToEndOfEventTime, String savepointDirectory, boolean nativeFormat) { diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java index 0af46b845a..eec452e830 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java @@ -23,7 +23,7 @@ import java.util.Optional; /** Flink 2.0 Kubernetes client. */ -public class FlinkKubernetesClient extends FlinkKubernetesClientTrait { +public class FlinkKubernetesClient extends AbstractFlinkKubernetesClient { public FlinkKubernetesClient(FlinkKubeClient kubeClient) { super(kubeClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index 31b6aba217..b16f616a53 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -17,235 +17,22 @@ package org.apache.streampark.flink.core; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.Schema; +import org.apache.streampark.flink.core.bean.StreamTableContextSpec; + import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableDescriptor; -import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.bridge.java.StreamStatementSet; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.table.catalog.CatalogDescriptor; -import org.apache.flink.table.connector.ChangelogMode; -import org.apache.flink.table.expressions.Expression; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; -import org.apache.flink.table.types.AbstractDataType; -import org.apache.flink.types.Row; -import org.apache.flink.util.ParameterTool; - -import java.util.List; /** Flink 2.0 stream-table environment context. */ -public class StreamTableContext extends FlinkStreamTableTraitV2 { - - public StreamTableContext( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment tableEnv) { - super(parameter, streamEnv, tableEnv); - } - - public StreamTableContext(FlinkTableInitializerV2.StreamTableInitResult init) { - this(init.parameter, init.streamEnv, init.streamTableEnv); - } - - public StreamTableContext(StreamTableEnvConfig config) { - this(FlinkTableInitializerV2.initialize(config)); - } - - @Override - public Table fromDataStream(DataStream dataStream, Schema schema) { - return getStreamTableEnv().fromDataStream(dataStream, schema); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public Table fromDataStream(DataStream dataStream, Expression... expressions) { - return getStreamTableEnv().fromDataStream(dataStream, expressions); - } - - @Override - public Table fromChangelogStream(DataStream dataStream) { - return getStreamTableEnv().fromChangelogStream(dataStream); - } - - @Override - public Table fromChangelogStream(DataStream dataStream, Schema schema) { - return getStreamTableEnv().fromChangelogStream(dataStream, schema); - } - - @Override - public Table fromChangelogStream( - DataStream dataStream, Schema schema, ChangelogMode changelogMode) { - return getStreamTableEnv().fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView(String path, DataStream dataStream, Schema schema) { - getStreamTableEnv().createTemporaryView(path, dataStream, schema); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void createTemporaryView( - String path, DataStream dataStream, Expression... expressions) { - getStreamTableEnv().createTemporaryView(path, dataStream, expressions); - } - - @Override - public DataStream toDataStream(Table table) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table); - } - - @Override - public DataStream toDataStream(Table table, Class targetClass) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table, targetClass); - } - - @Override - public DataStream toDataStream(Table table, AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table, targetDataType); - } - - @Override - public DataStream toChangelogStream(Table table) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table); - } - - @Override - public DataStream toChangelogStream(Table table, Schema targetSchema) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table, targetSchema); - } - - @Override - public DataStream toChangelogStream( - Table table, Schema targetSchema, ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public StreamStatementSet createStatementSet() { - return getStreamTableEnv().createStatementSet(); - } - - @Override - public void useModules(String... moduleNames) { - getStreamTableEnv().useModules(moduleNames); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - getStreamTableEnv().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - getStreamTableEnv().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor descriptor) { - return getStreamTableEnv().from(descriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return getStreamTableEnv().listFullModules(); - } - - @Override - public String[] listTables(String catalogName, String databaseName) { - return getStreamTableEnv().listTables(catalogName, databaseName); - } - - @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return getStreamTableEnv().loadPlan(planReference); - } - - @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return getStreamTableEnv().compilePlanSql(statement); - } - - @Override - public void createFunction(String path, String className, List resourceUris) { - getStreamTableEnv().createFunction(path, className, resourceUris); - } - - @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - getStreamTableEnv().createFunction(path, className, resourceUris, ignoreIfExists); - } +public class StreamTableContext extends AbstractFlinkStreamTable { - @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - getStreamTableEnv().createTemporaryFunction(path, className, resourceUris); - } - - @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - getStreamTableEnv().createTemporarySystemFunction(name, className, resourceUris); - } - - @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return getStreamTableEnv().explainSql(statement, format, extraDetails); + public StreamTableContext(StreamTableContextSpec streamContextConfig) { + super(streamContextConfig.parameter, streamContextConfig.streamEnv, streamContextConfig.streamTableEnv); } @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - getStreamTableEnv().createCatalog(catalogName, catalogDescriptor); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public DataStream toAppendStream(Table table, TypeInformation typeInformation) { - isConvertedToDataStream = true; - return getStreamTableEnv().toAppendStream(table, typeInformation); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public DataStream> toRetractStream( - Table table, TypeInformation typeInformation) { - isConvertedToDataStream = true; - return getStreamTableEnv().toRetractStream(table, typeInformation); - } - - @Override - public DataStream toAppendStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return getStreamTableEnv().toAppendStream(table, clazz); - } - - @Override - public DataStream> toRetractStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return getStreamTableEnv().toRetractStream(table, clazz); + public void createTemporaryTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + getStreamTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); } @Override @@ -254,9 +41,8 @@ public boolean createTable(String path, TableDescriptor descriptor, boolean igno } @Override - public void createTemporaryTable( - String path, TableDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); + public void createView(String path, Table view) { + getStreamTableEnv().createView(path, view); } @Override @@ -265,8 +51,8 @@ public boolean createView(String path, Table view, boolean ignoreIfExists) { } @Override - public void createView(String path, Table view) { - getStreamTableEnv().createView(path, view); + public boolean dropTable(String path) { + return getStreamTableEnv().dropTable(path); } @Override @@ -275,8 +61,8 @@ public boolean dropTable(String path, boolean ignoreIfNotExists) { } @Override - public boolean dropTable(String path) { - return getStreamTableEnv().dropTable(path); + public boolean dropView(String path) { + return getStreamTableEnv().dropView(path); } @Override @@ -285,7 +71,8 @@ public boolean dropView(String path, boolean ignoreIfNotExists) { } @Override - public boolean dropView(String path) { - return getStreamTableEnv().dropView(path); + public StreamStatementSet createStatementSet() { + return getStreamTableEnv().createStatementSet(); } + } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/TableContext.java index 7418d924fe..6b8a9a576d 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -17,110 +17,27 @@ package org.apache.streampark.flink.core; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; -import org.apache.flink.table.api.PlanReference; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.core.bean.TableContextSpec; + import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.TableException; -import org.apache.flink.table.catalog.CatalogDescriptor; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; -import org.apache.flink.util.ParameterTool; - -import java.util.List; /** Flink 2.0 table environment context. */ -public class TableContext extends FlinkTableTrait { +public class TableContext extends AbstractFlinkTable { - public TableContext(ParameterTool parameter, TableEnvironment tableEnv) { + public TableContext(FlinkJobParameters parameter, TableEnvironment tableEnv) { super(parameter, tableEnv); } - public TableContext(FlinkTableInitializerV2.TableInitResult init) { - this(init.parameter, init.tableEnv); - } - - public TableContext(TableEnvConfig config) { - this(FlinkTableInitializerV2.initialize(config)); - } - - @Override - public void useModules(String... moduleNames) { - getTableEnv().useModules(moduleNames); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - getTableEnv().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - getTableEnv().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor descriptor) { - return getTableEnv().from(descriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return getTableEnv().listFullModules(); - } - - @Override - public String[] listTables(String catalogName, String databaseName) { - return getTableEnv().listTables(catalogName, databaseName); - } - - @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return getTableEnv().loadPlan(planReference); - } - - @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return getTableEnv().compilePlanSql(statement); - } - - @Override - public void createFunction(String path, String className, List resourceUris) { - getTableEnv().createFunction(path, className, resourceUris); - } - - @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - getTableEnv().createFunction(path, className, resourceUris, ignoreIfExists); + public TableContext(TableContextSpec contextConfig) { + this(contextConfig.parameter, contextConfig.tableEnv); } @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - getTableEnv().createTemporaryFunction(path, className, resourceUris); - } - - @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - getTableEnv().createTemporarySystemFunction(name, className, resourceUris); - } - - @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return getTableEnv().explainSql(statement, format, extraDetails); - } - - @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - getTableEnv().createCatalog(catalogName, catalogDescriptor); + public void createTemporaryTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + getTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); } @Override @@ -129,9 +46,8 @@ public boolean createTable(String path, TableDescriptor descriptor, boolean igno } @Override - public void createTemporaryTable( - String path, TableDescriptor descriptor, boolean ignoreIfExists) { - getTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); + public void createView(String path, Table view) { + getTableEnv().createView(path, view); } @Override @@ -140,8 +56,8 @@ public boolean createView(String path, Table view, boolean ignoreIfExists) { } @Override - public void createView(String path, Table view) { - getTableEnv().createView(path, view); + public boolean dropTable(String path) { + return getTableEnv().dropTable(path); } @Override @@ -150,17 +66,12 @@ public boolean dropTable(String path, boolean ignoreIfNotExists) { } @Override - public boolean dropTable(String path) { - return getTableEnv().dropTable(path); + public boolean dropView(String path) { + return getTableEnv().dropView(path); } @Override public boolean dropView(String path, boolean ignoreIfNotExists) { return getTableEnv().dropView(path, ignoreIfNotExists); } - - @Override - public boolean dropView(String path) { - return getTableEnv().dropView(path); - } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/TableExt.java deleted file mode 100644 index e72fb5a70c..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.0/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.types.Row; - -/** Table API extensions for Flink 2.0 Java stream-table applications. */ -public final class TableExt { - - private TableExt() { - } - - /** Table alias helper (Scala {@code ->} operator equivalent: {@code as}). */ - public static final class Table { - - private final org.apache.flink.table.api.Table flinkTable; - - public Table(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - public org.apache.flink.table.api.Table as(String field, String... fields) { - return flinkTable.as(field, fields); - } - } - - /** Table-to-DataStream conversion helpers. */ - public static class TableConversions { - - private final org.apache.flink.table.api.Table flinkTable; - - private final StreamTableEnvironment streamTableEnv; - - public TableConversions( - org.apache.flink.table.api.Table table, StreamTableEnvironment streamTableEnv) { - this.flinkTable = table; - this.streamTableEnv = streamTableEnv; - } - - /** Changelog stream conversion (Scala {@code \\} operator equivalent). */ - public DataStream toChangelogDataStream() { - return streamTableEnv.toDataStream(flinkTable); - } - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/pom.xml index d85e1cdf0a..c54669575e 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/pom.xml +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/pom.xml @@ -35,7 +35,7 @@ org.apache.streampark - streampark-flink-shims-base-v2 + streampark-flink-shims-base ${project.version} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java index 5378f12170..21fe200fe8 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java @@ -24,7 +24,7 @@ import java.util.concurrent.CompletableFuture; /** Flink 2.1 cluster client with native/canonical savepoint format support. */ -public class FlinkClusterClient extends FlinkClientTrait { +public class FlinkClusterClient extends AbstractFlinkClient { public FlinkClusterClient(ClusterClient clusterClient) { super(clusterClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java index fc40fb8f42..1e0f2b9e52 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java @@ -23,7 +23,7 @@ import java.util.Optional; /** Flink 2.1 Kubernetes client. */ -public class FlinkKubernetesClient extends FlinkKubernetesClientTrait { +public class FlinkKubernetesClient extends AbstractFlinkKubernetesClient { public FlinkKubernetesClient(FlinkKubeClient kubeClient) { super(kubeClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index 781927abec..1d91b4783e 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -17,263 +17,79 @@ package org.apache.streampark.flink.core; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; +import org.apache.streampark.flink.core.bean.StreamTableContextSpec; + import org.apache.flink.table.api.ModelDescriptor; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.Schema; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableDescriptor; -import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.bridge.java.StreamStatementSet; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.table.catalog.CatalogDescriptor; -import org.apache.flink.table.connector.ChangelogMode; -import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.functions.UserDefinedFunction; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; -import org.apache.flink.table.types.AbstractDataType; -import org.apache.flink.types.Row; -import org.apache.flink.util.ParameterTool; - -import java.util.List; /** Flink 2.1 stream-table environment context. */ -public class StreamTableContext extends FlinkStreamTableTraitV2 { - - public StreamTableContext( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment tableEnv) { - super(parameter, streamEnv, tableEnv); - } - - public StreamTableContext(FlinkTableInitializerV2.StreamTableInitResult init) { - this(init.parameter, init.streamEnv, init.streamTableEnv); - } - - public StreamTableContext(StreamTableEnvConfig config) { - this(FlinkTableInitializerV2.initialize(config)); - } - - @Override - public Table fromDataStream(DataStream dataStream, Schema schema) { - return getStreamTableEnv().fromDataStream(dataStream, schema); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public Table fromDataStream(DataStream dataStream, Expression... expressions) { - return getStreamTableEnv().fromDataStream(dataStream, expressions); - } - - @Override - public Table fromChangelogStream(DataStream dataStream) { - return getStreamTableEnv().fromChangelogStream(dataStream); - } - - @Override - public Table fromChangelogStream(DataStream dataStream, Schema schema) { - return getStreamTableEnv().fromChangelogStream(dataStream, schema); - } - - @Override - public Table fromChangelogStream( - DataStream dataStream, Schema schema, ChangelogMode changelogMode) { - return getStreamTableEnv().fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView(String path, DataStream dataStream, Schema schema) { - getStreamTableEnv().createTemporaryView(path, dataStream, schema); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void createTemporaryView( - String path, DataStream dataStream, Expression... expressions) { - getStreamTableEnv().createTemporaryView(path, dataStream, expressions); - } - - @Override - public DataStream toDataStream(Table table) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table); - } - - @Override - public DataStream toDataStream(Table table, Class targetClass) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table, targetClass); - } - - @Override - public DataStream toDataStream(Table table, AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table, targetDataType); - } - - @Override - public DataStream toChangelogStream(Table table) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table); - } - - @Override - public DataStream toChangelogStream(Table table, Schema targetSchema) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table, targetSchema); - } - - @Override - public DataStream toChangelogStream( - Table table, Schema targetSchema, ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public StreamStatementSet createStatementSet() { - return getStreamTableEnv().createStatementSet(); - } - - @Override - public void useModules(String... moduleNames) { - getStreamTableEnv().useModules(moduleNames); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - getStreamTableEnv().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - getStreamTableEnv().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor descriptor) { - return getStreamTableEnv().from(descriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return getStreamTableEnv().listFullModules(); - } - - @Override - public String[] listTables(String catalogName, String databaseName) { - return getStreamTableEnv().listTables(catalogName, databaseName); - } - - @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return getStreamTableEnv().loadPlan(planReference); - } - - @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return getStreamTableEnv().compilePlanSql(statement); - } - - @Override - public void createFunction(String path, String className, List resourceUris) { - getStreamTableEnv().createFunction(path, className, resourceUris); - } - - @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - getStreamTableEnv().createFunction(path, className, resourceUris, ignoreIfExists); - } +public class StreamTableContext extends AbstractFlinkStreamTable { - @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - getStreamTableEnv().createTemporaryFunction(path, className, resourceUris); + public StreamTableContext(StreamTableContextSpec streamContextConfig) { + super(streamContextConfig.parameter, streamContextConfig.streamEnv, streamContextConfig.streamTableEnv); } @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - getStreamTableEnv().createTemporarySystemFunction(name, className, resourceUris); + public void createTemporaryTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + getStreamTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); } @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return getStreamTableEnv().explainSql(statement, format, extraDetails); + public boolean createTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + return getStreamTableEnv().createTable(path, descriptor, ignoreIfExists); } @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - getStreamTableEnv().createCatalog(catalogName, catalogDescriptor); + public void createView(String path, Table view) { + getStreamTableEnv().createView(path, view); } - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) @Override - public DataStream toAppendStream(Table table, TypeInformation typeInformation) { - isConvertedToDataStream = true; - return getStreamTableEnv().toAppendStream(table, typeInformation); + public boolean createView(String path, Table view, boolean ignoreIfExists) { + return getStreamTableEnv().createView(path, view, ignoreIfExists); } - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) @Override - public DataStream> toRetractStream( - Table table, TypeInformation typeInformation) { - isConvertedToDataStream = true; - return getStreamTableEnv().toRetractStream(table, typeInformation); + public void createModel(String path, ModelDescriptor descriptor) { + getStreamTableEnv().createModel(path, descriptor); } @Override - public DataStream toAppendStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return getStreamTableEnv().toAppendStream(table, clazz); + public void createModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { + getStreamTableEnv().createModel(path, descriptor, ignoreIfExists); } @Override - public DataStream> toRetractStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return getStreamTableEnv().toRetractStream(table, clazz); + public void createTemporaryModel(String path, ModelDescriptor descriptor) { + getStreamTableEnv().createTemporaryModel(path, descriptor); } @Override - public boolean createTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { - return getStreamTableEnv().createTable(path, descriptor, ignoreIfExists); + public void createTemporaryModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { + getStreamTableEnv().createTemporaryModel(path, descriptor, ignoreIfExists); } @Override - public void createTemporaryTable( - String path, TableDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); + public Table fromCall(String path, Object... arguments) { + return getStreamTableEnv().fromCall(path, arguments); } @Override - public boolean createView(String path, Table view, boolean ignoreIfExists) { - return getStreamTableEnv().createView(path, view, ignoreIfExists); + public Table fromCall(Class function, Object... arguments) { + return getStreamTableEnv().fromCall(function, arguments); } @Override - public void createView(String path, Table view) { - getStreamTableEnv().createView(path, view); + public String[] listModels() { + return getStreamTableEnv().listModels(); } @Override - public boolean dropTable(String path, boolean ignoreIfNotExists) { - return getStreamTableEnv().dropTable(path, ignoreIfNotExists); + public String[] listTemporaryModels() { + return getStreamTableEnv().listTemporaryModels(); } @Override @@ -282,8 +98,8 @@ public boolean dropTable(String path) { } @Override - public boolean dropView(String path, boolean ignoreIfNotExists) { - return getStreamTableEnv().dropView(path, ignoreIfNotExists); + public boolean dropTable(String path, boolean ignoreIfNotExists) { + return getStreamTableEnv().dropTable(path, ignoreIfNotExists); } @Override @@ -292,24 +108,13 @@ public boolean dropView(String path) { } @Override - public void createModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createModel(path, descriptor, ignoreIfExists); - } - - @Override - public void createModel(String path, ModelDescriptor descriptor) { - getStreamTableEnv().createModel(path, descriptor); - } - - @Override - public void createTemporaryModel( - String path, ModelDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createTemporaryModel(path, descriptor, ignoreIfExists); + public boolean dropView(String path, boolean ignoreIfNotExists) { + return getStreamTableEnv().dropView(path, ignoreIfNotExists); } @Override - public void createTemporaryModel(String path, ModelDescriptor descriptor) { - getStreamTableEnv().createTemporaryModel(path, descriptor); + public boolean dropModel(String path) { + return getStreamTableEnv().dropModel(path); } @Override @@ -317,33 +122,14 @@ public boolean dropModel(String path, boolean ignoreIfNotExists) { return getStreamTableEnv().dropModel(path, ignoreIfNotExists); } - @Override - public boolean dropModel(String path) { - return getStreamTableEnv().dropModel(path); - } - @Override public boolean dropTemporaryModel(String path) { return getStreamTableEnv().dropTemporaryModel(path); } @Override - public Table fromCall(Class functionClass, Object... arguments) { - return getStreamTableEnv().fromCall(functionClass, arguments); - } - - @Override - public Table fromCall(String functionName, Object... arguments) { - return getStreamTableEnv().fromCall(functionName, arguments); - } - - @Override - public String[] listModels() { - return getStreamTableEnv().listModels(); + public StreamStatementSet createStatementSet() { + return getStreamTableEnv().createStatementSet(); } - @Override - public String[] listTemporaryModels() { - return getStreamTableEnv().listTemporaryModels(); - } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/TableContext.java index f0862ee28e..d163be3d97 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -17,138 +17,84 @@ package org.apache.streampark.flink.core; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.core.bean.TableContextSpec; + import org.apache.flink.table.api.ModelDescriptor; -import org.apache.flink.table.api.PlanReference; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.TableException; -import org.apache.flink.table.catalog.CatalogDescriptor; import org.apache.flink.table.functions.UserDefinedFunction; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; -import org.apache.flink.util.ParameterTool; - -import java.util.List; /** Flink 2.1 table environment context. */ -public class TableContext extends FlinkTableTrait { +public class TableContext extends AbstractFlinkTable { - public TableContext(ParameterTool parameter, TableEnvironment tableEnv) { + public TableContext(FlinkJobParameters parameter, TableEnvironment tableEnv) { super(parameter, tableEnv); } - public TableContext(FlinkTableInitializerV2.TableInitResult init) { - this(init.parameter, init.tableEnv); - } - - public TableContext(TableEnvConfig config) { - this(FlinkTableInitializerV2.initialize(config)); - } - - @Override - public void useModules(String... moduleNames) { - getTableEnv().useModules(moduleNames); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - getTableEnv().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - getTableEnv().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor descriptor) { - return getTableEnv().from(descriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return getTableEnv().listFullModules(); - } - - @Override - public String[] listTables(String catalogName, String databaseName) { - return getTableEnv().listTables(catalogName, databaseName); + public TableContext(TableContextSpec contextConfig) { + this(contextConfig.parameter, contextConfig.tableEnv); } @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return getTableEnv().loadPlan(planReference); - } - - @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return getTableEnv().compilePlanSql(statement); + public void createTemporaryTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + getTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); } @Override - public void createFunction(String path, String className, List resourceUris) { - getTableEnv().createFunction(path, className, resourceUris); + public boolean createTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + return getTableEnv().createTable(path, descriptor, ignoreIfExists); } @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - getTableEnv().createFunction(path, className, resourceUris, ignoreIfExists); + public void createView(String path, Table view) { + getTableEnv().createView(path, view); } @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - getTableEnv().createTemporaryFunction(path, className, resourceUris); + public boolean createView(String path, Table view, boolean ignoreIfExists) { + return getTableEnv().createView(path, view, ignoreIfExists); } @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - getTableEnv().createTemporarySystemFunction(name, className, resourceUris); + public void createModel(String path, ModelDescriptor descriptor) { + getTableEnv().createModel(path, descriptor); } @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return getTableEnv().explainSql(statement, format, extraDetails); + public void createModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { + getTableEnv().createModel(path, descriptor, ignoreIfExists); } @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - getTableEnv().createCatalog(catalogName, catalogDescriptor); + public void createTemporaryModel(String path, ModelDescriptor descriptor) { + getTableEnv().createTemporaryModel(path, descriptor); } @Override - public boolean createTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { - return getTableEnv().createTable(path, descriptor, ignoreIfExists); + public void createTemporaryModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { + getTableEnv().createTemporaryModel(path, descriptor, ignoreIfExists); } @Override - public void createTemporaryTable( - String path, TableDescriptor descriptor, boolean ignoreIfExists) { - getTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); + public Table fromCall(String path, Object... arguments) { + return getTableEnv().fromCall(path, arguments); } @Override - public boolean createView(String path, Table view, boolean ignoreIfExists) { - return getTableEnv().createView(path, view, ignoreIfExists); + public Table fromCall(Class function, Object... arguments) { + return getTableEnv().fromCall(function, arguments); } @Override - public void createView(String path, Table view) { - getTableEnv().createView(path, view); + public String[] listModels() { + return getTableEnv().listModels(); } @Override - public boolean dropTable(String path, boolean ignoreIfNotExists) { - return getTableEnv().dropTable(path, ignoreIfNotExists); + public String[] listTemporaryModels() { + return getTableEnv().listTemporaryModels(); } @Override @@ -157,8 +103,8 @@ public boolean dropTable(String path) { } @Override - public boolean dropView(String path, boolean ignoreIfNotExists) { - return getTableEnv().dropView(path, ignoreIfNotExists); + public boolean dropTable(String path, boolean ignoreIfNotExists) { + return getTableEnv().dropTable(path, ignoreIfNotExists); } @Override @@ -167,24 +113,13 @@ public boolean dropView(String path) { } @Override - public void createModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { - getTableEnv().createModel(path, descriptor, ignoreIfExists); - } - - @Override - public void createModel(String path, ModelDescriptor descriptor) { - getTableEnv().createModel(path, descriptor); - } - - @Override - public void createTemporaryModel( - String path, ModelDescriptor descriptor, boolean ignoreIfExists) { - getTableEnv().createTemporaryModel(path, descriptor, ignoreIfExists); + public boolean dropView(String path, boolean ignoreIfNotExists) { + return getTableEnv().dropView(path, ignoreIfNotExists); } @Override - public void createTemporaryModel(String path, ModelDescriptor descriptor) { - getTableEnv().createTemporaryModel(path, descriptor); + public boolean dropModel(String path) { + return getTableEnv().dropModel(path); } @Override @@ -192,33 +127,8 @@ public boolean dropModel(String path, boolean ignoreIfNotExists) { return getTableEnv().dropModel(path, ignoreIfNotExists); } - @Override - public boolean dropModel(String path) { - return getTableEnv().dropModel(path); - } - @Override public boolean dropTemporaryModel(String path) { return getTableEnv().dropTemporaryModel(path); } - - @Override - public Table fromCall(Class functionClass, Object... arguments) { - return getTableEnv().fromCall(functionClass, arguments); - } - - @Override - public Table fromCall(String functionName, Object... arguments) { - return getTableEnv().fromCall(functionName, arguments); - } - - @Override - public String[] listModels() { - return getTableEnv().listModels(); - } - - @Override - public String[] listTemporaryModels() { - return getTableEnv().listTemporaryModels(); - } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/TableExt.java deleted file mode 100644 index 3464082976..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.1/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.types.Row; - -/** Table API extensions for Flink 2.1 Java stream-table applications. */ -public final class TableExt { - - private TableExt() { - } - - /** Table alias helper (Scala {@code ->} operator equivalent: {@code as}). */ - public static final class Table { - - private final org.apache.flink.table.api.Table flinkTable; - - public Table(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - public org.apache.flink.table.api.Table as(String field, String... fields) { - return flinkTable.as(field, fields); - } - } - - /** Table-to-DataStream conversion helpers. */ - public static class TableConversions { - - private final org.apache.flink.table.api.Table flinkTable; - - private final StreamTableEnvironment streamTableEnv; - - public TableConversions( - org.apache.flink.table.api.Table table, StreamTableEnvironment streamTableEnv) { - this.flinkTable = table; - this.streamTableEnv = streamTableEnv; - } - - /** Changelog stream conversion (Scala {@code \\} operator equivalent). */ - public DataStream toChangelogDataStream() { - return streamTableEnv.toDataStream(flinkTable); - } - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/pom.xml index e41613f823..2d8830fa59 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/pom.xml +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/pom.xml @@ -35,7 +35,7 @@ org.apache.streampark - streampark-flink-shims-base-v2 + streampark-flink-shims-base ${project.version} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java index 38a97e7ffd..18f4bd00ac 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java @@ -24,7 +24,7 @@ import java.util.concurrent.CompletableFuture; /** Flink 2.2 cluster client with native/canonical savepoint format support. */ -public class FlinkClusterClient extends FlinkClientTrait { +public class FlinkClusterClient extends AbstractFlinkClient { public FlinkClusterClient(ClusterClient clusterClient) { super(clusterClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java index 069e2b37c4..3aa94f019f 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java @@ -23,7 +23,7 @@ import java.util.Optional; /** Flink 2.2 Kubernetes client. */ -public class FlinkKubernetesClient extends FlinkKubernetesClientTrait { +public class FlinkKubernetesClient extends AbstractFlinkKubernetesClient { public FlinkKubernetesClient(FlinkKubeClient kubeClient) { super(kubeClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index 0ab688c846..c0870c73bb 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -17,265 +17,116 @@ package org.apache.streampark.flink.core; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; +import org.apache.streampark.flink.core.bean.StreamTableContextSpec; + import org.apache.flink.table.api.FunctionDescriptor; import org.apache.flink.table.api.Model; import org.apache.flink.table.api.ModelDescriptor; -import org.apache.flink.table.api.PlanReference; -import org.apache.flink.table.api.Schema; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableDescriptor; -import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.bridge.java.StreamStatementSet; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.table.catalog.CatalogDescriptor; -import org.apache.flink.table.connector.ChangelogMode; -import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.functions.UserDefinedFunction; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; -import org.apache.flink.table.types.AbstractDataType; -import org.apache.flink.types.Row; -import org.apache.flink.util.ParameterTool; - -import java.util.List; /** Flink 2.2 stream-table environment context. */ -public class StreamTableContext extends FlinkStreamTableTraitV2 { - - public StreamTableContext( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment tableEnv) { - super(parameter, streamEnv, tableEnv); - } - - public StreamTableContext(FlinkTableInitializerV2.StreamTableInitResult init) { - this(init.parameter, init.streamEnv, init.streamTableEnv); - } - - public StreamTableContext(StreamTableEnvConfig config) { - this(FlinkTableInitializerV2.initialize(config)); - } - - @Override - public Table fromDataStream(DataStream dataStream, Schema schema) { - return getStreamTableEnv().fromDataStream(dataStream, schema); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public Table fromDataStream(DataStream dataStream, Expression... expressions) { - return getStreamTableEnv().fromDataStream(dataStream, expressions); - } - - @Override - public Table fromChangelogStream(DataStream dataStream) { - return getStreamTableEnv().fromChangelogStream(dataStream); - } - - @Override - public Table fromChangelogStream(DataStream dataStream, Schema schema) { - return getStreamTableEnv().fromChangelogStream(dataStream, schema); - } +public class StreamTableContext extends AbstractFlinkStreamTable { - @Override - public Table fromChangelogStream( - DataStream dataStream, Schema schema, ChangelogMode changelogMode) { - return getStreamTableEnv().fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView(String path, DataStream dataStream, Schema schema) { - getStreamTableEnv().createTemporaryView(path, dataStream, schema); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void createTemporaryView( - String path, DataStream dataStream, Expression... expressions) { - getStreamTableEnv().createTemporaryView(path, dataStream, expressions); - } - - @Override - public DataStream toDataStream(Table table) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table); + public StreamTableContext(StreamTableContextSpec streamContextConfig) { + super(streamContextConfig.parameter, streamContextConfig.streamEnv, streamContextConfig.streamTableEnv); } @Override - public DataStream toDataStream(Table table, Class targetClass) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table, targetClass); + public void createFunction(String path, FunctionDescriptor functionDescriptor) { + getStreamTableEnv().createFunction(path, functionDescriptor); } @Override - public DataStream toDataStream(Table table, AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table, targetDataType); + public void createFunction(String path, FunctionDescriptor functionDescriptor, boolean ignoreIfExists) { + getStreamTableEnv().createFunction(path, functionDescriptor, ignoreIfExists); } @Override - public DataStream toChangelogStream(Table table) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table); + public void createTemporaryFunction(String path, FunctionDescriptor functionDescriptor) { + getStreamTableEnv().createTemporaryFunction(path, functionDescriptor); } @Override - public DataStream toChangelogStream(Table table, Schema targetSchema) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table, targetSchema); + public void createTemporarySystemFunction(String name, FunctionDescriptor functionDescriptor) { + getStreamTableEnv().createTemporarySystemFunction(name, functionDescriptor); } @Override - public DataStream toChangelogStream( - Table table, Schema targetSchema, ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public StreamStatementSet createStatementSet() { - return getStreamTableEnv().createStatementSet(); - } - - @Override - public void useModules(String... moduleNames) { - getStreamTableEnv().useModules(moduleNames); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - getStreamTableEnv().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - getStreamTableEnv().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor descriptor) { - return getStreamTableEnv().from(descriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return getStreamTableEnv().listFullModules(); - } - - @Override - public String[] listTables(String catalogName, String databaseName) { - return getStreamTableEnv().listTables(catalogName, databaseName); - } - - @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return getStreamTableEnv().loadPlan(planReference); - } - - @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return getStreamTableEnv().compilePlanSql(statement); - } - - @Override - public void createFunction(String path, String className, List resourceUris) { - getStreamTableEnv().createFunction(path, className, resourceUris); + public void createTemporaryTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + getStreamTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); } @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - getStreamTableEnv().createFunction(path, className, resourceUris, ignoreIfExists); + public boolean createTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + return getStreamTableEnv().createTable(path, descriptor, ignoreIfExists); } @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - getStreamTableEnv().createTemporaryFunction(path, className, resourceUris); + public void createView(String path, Table view) { + getStreamTableEnv().createView(path, view); } @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - getStreamTableEnv().createTemporarySystemFunction(name, className, resourceUris); + public boolean createView(String path, Table view, boolean ignoreIfExists) { + return getStreamTableEnv().createView(path, view, ignoreIfExists); } @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return getStreamTableEnv().explainSql(statement, format, extraDetails); + public void createModel(String path, ModelDescriptor descriptor) { + getStreamTableEnv().createModel(path, descriptor); } @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - getStreamTableEnv().createCatalog(catalogName, catalogDescriptor); + public void createModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { + getStreamTableEnv().createModel(path, descriptor, ignoreIfExists); } - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) @Override - public DataStream toAppendStream(Table table, TypeInformation typeInformation) { - isConvertedToDataStream = true; - return getStreamTableEnv().toAppendStream(table, typeInformation); + public void createTemporaryModel(String path, ModelDescriptor descriptor) { + getStreamTableEnv().createTemporaryModel(path, descriptor); } - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) @Override - public DataStream> toRetractStream( - Table table, TypeInformation typeInformation) { - isConvertedToDataStream = true; - return getStreamTableEnv().toRetractStream(table, typeInformation); + public void createTemporaryModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { + getStreamTableEnv().createTemporaryModel(path, descriptor, ignoreIfExists); } @Override - public DataStream toAppendStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return getStreamTableEnv().toAppendStream(table, clazz); + public Table fromCall(String path, Object... arguments) { + return getStreamTableEnv().fromCall(path, arguments); } @Override - public DataStream> toRetractStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return getStreamTableEnv().toRetractStream(table, clazz); + public Table fromCall(Class function, Object... arguments) { + return getStreamTableEnv().fromCall(function, arguments); } @Override - public boolean createTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { - return getStreamTableEnv().createTable(path, descriptor, ignoreIfExists); + public Model fromModel(String modelPath) { + return getStreamTableEnv().fromModel(modelPath); } @Override - public void createTemporaryTable( - String path, TableDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); + public Model fromModel(ModelDescriptor descriptor) { + return getStreamTableEnv().fromModel(descriptor); } @Override - public boolean createView(String path, Table view, boolean ignoreIfExists) { - return getStreamTableEnv().createView(path, view, ignoreIfExists); + public String[] listMaterializedTables() { + return getStreamTableEnv().listMaterializedTables(); } @Override - public void createView(String path, Table view) { - getStreamTableEnv().createView(path, view); + public String[] listModels() { + return getStreamTableEnv().listModels(); } @Override - public boolean dropTable(String path, boolean ignoreIfNotExists) { - return getStreamTableEnv().dropTable(path, ignoreIfNotExists); + public String[] listTemporaryModels() { + return getStreamTableEnv().listTemporaryModels(); } @Override @@ -284,8 +135,8 @@ public boolean dropTable(String path) { } @Override - public boolean dropView(String path, boolean ignoreIfNotExists) { - return getStreamTableEnv().dropView(path, ignoreIfNotExists); + public boolean dropTable(String path, boolean ignoreIfNotExists) { + return getStreamTableEnv().dropTable(path, ignoreIfNotExists); } @Override @@ -294,24 +145,13 @@ public boolean dropView(String path) { } @Override - public void createModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createModel(path, descriptor, ignoreIfExists); - } - - @Override - public void createModel(String path, ModelDescriptor descriptor) { - getStreamTableEnv().createModel(path, descriptor); - } - - @Override - public void createTemporaryModel( - String path, ModelDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createTemporaryModel(path, descriptor, ignoreIfExists); + public boolean dropView(String path, boolean ignoreIfNotExists) { + return getStreamTableEnv().dropView(path, ignoreIfNotExists); } @Override - public void createTemporaryModel(String path, ModelDescriptor descriptor) { - getStreamTableEnv().createTemporaryModel(path, descriptor); + public boolean dropModel(String path) { + return getStreamTableEnv().dropModel(path); } @Override @@ -319,69 +159,14 @@ public boolean dropModel(String path, boolean ignoreIfNotExists) { return getStreamTableEnv().dropModel(path, ignoreIfNotExists); } - @Override - public boolean dropModel(String path) { - return getStreamTableEnv().dropModel(path); - } - @Override public boolean dropTemporaryModel(String path) { return getStreamTableEnv().dropTemporaryModel(path); } @Override - public Table fromCall(Class functionClass, Object... arguments) { - return getStreamTableEnv().fromCall(functionClass, arguments); - } - - @Override - public Table fromCall(String functionName, Object... arguments) { - return getStreamTableEnv().fromCall(functionName, arguments); - } - - @Override - public String[] listModels() { - return getStreamTableEnv().listModels(); - } - - @Override - public String[] listTemporaryModels() { - return getStreamTableEnv().listTemporaryModels(); - } - - @Override - public void createFunction( - String path, FunctionDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createFunction(path, descriptor, ignoreIfExists); - } - - @Override - public void createFunction(String path, FunctionDescriptor descriptor) { - getStreamTableEnv().createFunction(path, descriptor); - } - - @Override - public void createTemporaryFunction(String path, FunctionDescriptor descriptor) { - getStreamTableEnv().createTemporaryFunction(path, descriptor); - } - - @Override - public void createTemporarySystemFunction(String name, FunctionDescriptor descriptor) { - getStreamTableEnv().createTemporarySystemFunction(name, descriptor); - } - - @Override - public Model fromModel(ModelDescriptor descriptor) { - return getStreamTableEnv().fromModel(descriptor); - } - - @Override - public Model fromModel(String path) { - return getStreamTableEnv().fromModel(path); + public StreamStatementSet createStatementSet() { + return getStreamTableEnv().createStatementSet(); } - @Override - public String[] listMaterializedTables() { - return getStreamTableEnv().listMaterializedTables(); - } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/TableContext.java index 3d7e3be304..bf04df13a7 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -17,114 +17,51 @@ package org.apache.streampark.flink.core; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.core.bean.TableContextSpec; + import org.apache.flink.table.api.FunctionDescriptor; import org.apache.flink.table.api.Model; import org.apache.flink.table.api.ModelDescriptor; -import org.apache.flink.table.api.PlanReference; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.TableException; -import org.apache.flink.table.catalog.CatalogDescriptor; import org.apache.flink.table.functions.UserDefinedFunction; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; -import org.apache.flink.util.ParameterTool; - -import java.util.List; /** Flink 2.2 table environment context. */ -public class TableContext extends FlinkTableTrait { +public class TableContext extends AbstractFlinkTable { - public TableContext(ParameterTool parameter, TableEnvironment tableEnv) { + public TableContext(FlinkJobParameters parameter, TableEnvironment tableEnv) { super(parameter, tableEnv); } - public TableContext(FlinkTableInitializerV2.TableInitResult init) { - this(init.parameter, init.tableEnv); - } - - public TableContext(TableEnvConfig config) { - this(FlinkTableInitializerV2.initialize(config)); - } - - @Override - public void useModules(String... moduleNames) { - getTableEnv().useModules(moduleNames); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - getTableEnv().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - getTableEnv().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor descriptor) { - return getTableEnv().from(descriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return getTableEnv().listFullModules(); - } - - @Override - public String[] listTables(String catalogName, String databaseName) { - return getTableEnv().listTables(catalogName, databaseName); - } - - @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return getTableEnv().loadPlan(planReference); - } - - @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return getTableEnv().compilePlanSql(statement); - } - - @Override - public void createFunction(String path, String className, List resourceUris) { - getTableEnv().createFunction(path, className, resourceUris); + public TableContext(TableContextSpec contextConfig) { + this(contextConfig.parameter, contextConfig.tableEnv); } @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - getTableEnv().createFunction(path, className, resourceUris, ignoreIfExists); + public void createFunction(String path, FunctionDescriptor functionDescriptor) { + getTableEnv().createFunction(path, functionDescriptor); } @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - getTableEnv().createTemporaryFunction(path, className, resourceUris); + public void createFunction(String path, FunctionDescriptor functionDescriptor, boolean ignoreIfExists) { + getTableEnv().createFunction(path, functionDescriptor, ignoreIfExists); } @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - getTableEnv().createTemporarySystemFunction(name, className, resourceUris); + public void createTemporaryFunction(String path, FunctionDescriptor functionDescriptor) { + getTableEnv().createTemporaryFunction(path, functionDescriptor); } @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return getTableEnv().explainSql(statement, format, extraDetails); + public void createTemporarySystemFunction(String name, FunctionDescriptor functionDescriptor) { + getTableEnv().createTemporarySystemFunction(name, functionDescriptor); } @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - getTableEnv().createCatalog(catalogName, catalogDescriptor); + public void createTemporaryTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + getTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); } @Override @@ -132,40 +69,19 @@ public boolean createTable(String path, TableDescriptor descriptor, boolean igno return getTableEnv().createTable(path, descriptor, ignoreIfExists); } - @Override - public void createTemporaryTable( - String path, TableDescriptor descriptor, boolean ignoreIfExists) { - getTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); - } - - @Override - public boolean createView(String path, Table view, boolean ignoreIfExists) { - return getTableEnv().createView(path, view, ignoreIfExists); - } - @Override public void createView(String path, Table view) { getTableEnv().createView(path, view); } @Override - public boolean dropTable(String path, boolean ignoreIfNotExists) { - return getTableEnv().dropTable(path, ignoreIfNotExists); - } - - @Override - public boolean dropTable(String path) { - return getTableEnv().dropTable(path); - } - - @Override - public boolean dropView(String path, boolean ignoreIfNotExists) { - return getTableEnv().dropView(path, ignoreIfNotExists); + public boolean createView(String path, Table view, boolean ignoreIfExists) { + return getTableEnv().createView(path, view, ignoreIfExists); } @Override - public boolean dropView(String path) { - return getTableEnv().dropView(path); + public void createModel(String path, ModelDescriptor descriptor) { + getTableEnv().createModel(path, descriptor); } @Override @@ -174,44 +90,38 @@ public void createModel(String path, ModelDescriptor descriptor, boolean ignoreI } @Override - public void createModel(String path, ModelDescriptor descriptor) { - getTableEnv().createModel(path, descriptor); + public void createTemporaryModel(String path, ModelDescriptor descriptor) { + getTableEnv().createTemporaryModel(path, descriptor); } @Override - public void createTemporaryModel( - String path, ModelDescriptor descriptor, boolean ignoreIfExists) { + public void createTemporaryModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { getTableEnv().createTemporaryModel(path, descriptor, ignoreIfExists); } @Override - public void createTemporaryModel(String path, ModelDescriptor descriptor) { - getTableEnv().createTemporaryModel(path, descriptor); - } - - @Override - public boolean dropModel(String path, boolean ignoreIfNotExists) { - return getTableEnv().dropModel(path, ignoreIfNotExists); + public Table fromCall(String path, Object... arguments) { + return getTableEnv().fromCall(path, arguments); } @Override - public boolean dropModel(String path) { - return getTableEnv().dropModel(path); + public Table fromCall(Class function, Object... arguments) { + return getTableEnv().fromCall(function, arguments); } @Override - public boolean dropTemporaryModel(String path) { - return getTableEnv().dropTemporaryModel(path); + public Model fromModel(String modelPath) { + return getTableEnv().fromModel(modelPath); } @Override - public Table fromCall(Class functionClass, Object... arguments) { - return getTableEnv().fromCall(functionClass, arguments); + public Model fromModel(ModelDescriptor descriptor) { + return getTableEnv().fromModel(descriptor); } @Override - public Table fromCall(String functionName, Object... arguments) { - return getTableEnv().fromCall(functionName, arguments); + public String[] listMaterializedTables() { + return getTableEnv().listMaterializedTables(); } @Override @@ -225,38 +135,37 @@ public String[] listTemporaryModels() { } @Override - public void createFunction( - String path, FunctionDescriptor descriptor, boolean ignoreIfExists) { - getTableEnv().createFunction(path, descriptor, ignoreIfExists); + public boolean dropTable(String path) { + return getTableEnv().dropTable(path); } @Override - public void createFunction(String path, FunctionDescriptor descriptor) { - getTableEnv().createFunction(path, descriptor); + public boolean dropTable(String path, boolean ignoreIfNotExists) { + return getTableEnv().dropTable(path, ignoreIfNotExists); } @Override - public void createTemporaryFunction(String path, FunctionDescriptor descriptor) { - getTableEnv().createTemporaryFunction(path, descriptor); + public boolean dropView(String path) { + return getTableEnv().dropView(path); } @Override - public void createTemporarySystemFunction(String name, FunctionDescriptor descriptor) { - getTableEnv().createTemporarySystemFunction(name, descriptor); + public boolean dropView(String path, boolean ignoreIfNotExists) { + return getTableEnv().dropView(path, ignoreIfNotExists); } @Override - public Model fromModel(ModelDescriptor descriptor) { - return getTableEnv().fromModel(descriptor); + public boolean dropModel(String path) { + return getTableEnv().dropModel(path); } @Override - public Model fromModel(String path) { - return getTableEnv().fromModel(path); + public boolean dropModel(String path, boolean ignoreIfNotExists) { + return getTableEnv().dropModel(path, ignoreIfNotExists); } @Override - public String[] listMaterializedTables() { - return getTableEnv().listMaterializedTables(); + public boolean dropTemporaryModel(String path) { + return getTableEnv().dropTemporaryModel(path); } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/TableExt.java deleted file mode 100644 index b9ac435702..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.2/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.types.Row; - -/** Table API extensions for Flink 2.2 Java stream-table applications. */ -public final class TableExt { - - private TableExt() { - } - - /** Table alias helper (Scala {@code ->} operator equivalent: {@code as}). */ - public static final class Table { - - private final org.apache.flink.table.api.Table flinkTable; - - public Table(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - public org.apache.flink.table.api.Table as(String field, String... fields) { - return flinkTable.as(field, fields); - } - } - - /** Table-to-DataStream conversion helpers. */ - public static class TableConversions { - - private final org.apache.flink.table.api.Table flinkTable; - - private final StreamTableEnvironment streamTableEnv; - - public TableConversions( - org.apache.flink.table.api.Table table, StreamTableEnvironment streamTableEnv) { - this.flinkTable = table; - this.streamTableEnv = streamTableEnv; - } - - /** Changelog stream conversion (Scala {@code \\} operator equivalent). */ - public DataStream toChangelogDataStream() { - return streamTableEnv.toDataStream(flinkTable); - } - } -} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/pom.xml b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/pom.xml index ac8db0f552..50907462c0 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/pom.xml +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/pom.xml @@ -35,7 +35,7 @@ org.apache.streampark - streampark-flink-shims-base-v2 + streampark-flink-shims-base ${project.version} diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java index 2699231c6a..102a5767a6 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/FlinkClusterClient.java @@ -24,7 +24,7 @@ import java.util.concurrent.CompletableFuture; /** Flink 2.3 cluster client with native/canonical savepoint format support. */ -public class FlinkClusterClient extends FlinkClientTrait { +public class FlinkClusterClient extends AbstractFlinkClient { public FlinkClusterClient(ClusterClient clusterClient) { super(clusterClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java index 6ed4e8e164..3e83f8c735 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/FlinkKubernetesClient.java @@ -23,7 +23,7 @@ import java.util.Optional; /** Flink 2.3 Kubernetes client. */ -public class FlinkKubernetesClient extends FlinkKubernetesClientTrait { +public class FlinkKubernetesClient extends AbstractFlinkKubernetesClient { public FlinkKubernetesClient(FlinkKubeClient kubeClient) { super(kubeClient); diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java index 9f64534abb..8e7845a3f3 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/StreamTableContext.java @@ -17,277 +17,121 @@ package org.apache.streampark.flink.core; -import org.apache.flink.api.common.typeinfo.TypeInformation; -import org.apache.flink.api.java.tuple.Tuple2; +import org.apache.streampark.flink.core.bean.StreamTableContextSpec; + import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; import org.apache.flink.table.api.FunctionDescriptor; import org.apache.flink.table.api.InsertConflictStrategy; import org.apache.flink.table.api.Model; import org.apache.flink.table.api.ModelDescriptor; -import org.apache.flink.table.api.PlanReference; import org.apache.flink.table.api.Schema; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableDescriptor; -import org.apache.flink.table.api.TableException; import org.apache.flink.table.api.bridge.java.StreamStatementSet; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.table.catalog.CatalogDescriptor; import org.apache.flink.table.connector.ChangelogMode; -import org.apache.flink.table.expressions.Expression; import org.apache.flink.table.functions.UserDefinedFunction; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; -import org.apache.flink.table.types.AbstractDataType; import org.apache.flink.types.Row; -import org.apache.flink.util.ParameterTool; - -import java.util.List; /** Flink 2.3 stream-table environment context. */ -public class StreamTableContext extends FlinkStreamTableTraitV2 { - - public StreamTableContext( - ParameterTool parameter, - StreamExecutionEnvironment streamEnv, - StreamTableEnvironment tableEnv) { - super(parameter, streamEnv, tableEnv); - } - - public StreamTableContext(FlinkTableInitializerV2.StreamTableInitResult init) { - this(init.parameter, init.streamEnv, init.streamTableEnv); - } - - public StreamTableContext(StreamTableEnvConfig config) { - this(FlinkTableInitializerV2.initialize(config)); - } - - @Override - public Table fromDataStream(DataStream dataStream, Schema schema) { - return getStreamTableEnv().fromDataStream(dataStream, schema); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public Table fromDataStream(DataStream dataStream, Expression... expressions) { - return getStreamTableEnv().fromDataStream(dataStream, expressions); - } - - @Override - public Table fromChangelogStream(DataStream dataStream) { - return getStreamTableEnv().fromChangelogStream(dataStream); - } - - @Override - public Table fromChangelogStream(DataStream dataStream, Schema schema) { - return getStreamTableEnv().fromChangelogStream(dataStream, schema); - } - - @Override - public Table fromChangelogStream( - DataStream dataStream, Schema schema, ChangelogMode changelogMode) { - return getStreamTableEnv().fromChangelogStream(dataStream, schema, changelogMode); - } - - @Override - public void createTemporaryView(String path, DataStream dataStream, Schema schema) { - getStreamTableEnv().createTemporaryView(path, dataStream, schema); - } - - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) - @Override - public void createTemporaryView( - String path, DataStream dataStream, Expression... expressions) { - getStreamTableEnv().createTemporaryView(path, dataStream, expressions); - } - - @Override - public DataStream toDataStream(Table table) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table); - } - - @Override - public DataStream toDataStream(Table table, Class targetClass) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table, targetClass); - } - - @Override - public DataStream toDataStream(Table table, AbstractDataType targetDataType) { - isConvertedToDataStream = true; - return getStreamTableEnv().toDataStream(table, targetDataType); - } - - @Override - public DataStream toChangelogStream(Table table) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table); - } - - @Override - public DataStream toChangelogStream(Table table, Schema targetSchema) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table, targetSchema); - } - - @Override - public DataStream toChangelogStream( - Table table, Schema targetSchema, ChangelogMode changelogMode) { - isConvertedToDataStream = true; - return getStreamTableEnv().toChangelogStream(table, targetSchema, changelogMode); - } - - @Override - public DataStream toChangelogStream( - Table table, - Schema targetSchema, - ChangelogMode changelogMode, - InsertConflictStrategy conflictStrategy) { - isConvertedToDataStream = true; - return getStreamTableEnv() - .toChangelogStream(table, targetSchema, changelogMode, conflictStrategy); - } +public class StreamTableContext extends AbstractFlinkStreamTable { - @Override - public StreamStatementSet createStatementSet() { - return getStreamTableEnv().createStatementSet(); + public StreamTableContext(StreamTableContextSpec streamContextConfig) { + super(streamContextConfig.parameter, streamContextConfig.streamEnv, streamContextConfig.streamTableEnv); } @Override - public void useModules(String... moduleNames) { - getStreamTableEnv().useModules(moduleNames); + public void createFunction(String path, FunctionDescriptor functionDescriptor) { + getStreamTableEnv().createFunction(path, functionDescriptor); } @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - getStreamTableEnv().createTemporaryTable(path, descriptor); + public void createFunction(String path, FunctionDescriptor functionDescriptor, boolean ignoreIfExists) { + getStreamTableEnv().createFunction(path, functionDescriptor, ignoreIfExists); } @Override - public void createTable(String path, TableDescriptor descriptor) { - getStreamTableEnv().createTable(path, descriptor); + public void createTemporaryFunction(String path, FunctionDescriptor functionDescriptor) { + getStreamTableEnv().createTemporaryFunction(path, functionDescriptor); } @Override - public Table from(TableDescriptor descriptor) { - return getStreamTableEnv().from(descriptor); + public void createTemporarySystemFunction(String name, FunctionDescriptor functionDescriptor) { + getStreamTableEnv().createTemporarySystemFunction(name, functionDescriptor); } @Override - public ModuleEntry[] listFullModules() { - return getStreamTableEnv().listFullModules(); - } - - @Override - public String[] listTables(String catalogName, String databaseName) { - return getStreamTableEnv().listTables(catalogName, databaseName); - } - - @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return getStreamTableEnv().loadPlan(planReference); - } - - @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return getStreamTableEnv().compilePlanSql(statement); - } - - @Override - public void createFunction(String path, String className, List resourceUris) { - getStreamTableEnv().createFunction(path, className, resourceUris); + public void createTemporaryTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + getStreamTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); } @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - getStreamTableEnv().createFunction(path, className, resourceUris, ignoreIfExists); + public boolean createTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + return getStreamTableEnv().createTable(path, descriptor, ignoreIfExists); } @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - getStreamTableEnv().createTemporaryFunction(path, className, resourceUris); + public void createView(String path, Table view) { + getStreamTableEnv().createView(path, view); } @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - getStreamTableEnv().createTemporarySystemFunction(name, className, resourceUris); + public boolean createView(String path, Table view, boolean ignoreIfExists) { + return getStreamTableEnv().createView(path, view, ignoreIfExists); } @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return getStreamTableEnv().explainSql(statement, format, extraDetails); + public void createModel(String path, ModelDescriptor descriptor) { + getStreamTableEnv().createModel(path, descriptor); } @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - getStreamTableEnv().createCatalog(catalogName, catalogDescriptor); + public void createModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { + getStreamTableEnv().createModel(path, descriptor, ignoreIfExists); } - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) @Override - public DataStream toAppendStream(Table table, TypeInformation typeInformation) { - isConvertedToDataStream = true; - return getStreamTableEnv().toAppendStream(table, typeInformation); + public void createTemporaryModel(String path, ModelDescriptor descriptor) { + getStreamTableEnv().createTemporaryModel(path, descriptor); } - /** @deprecated Retained for backward compatibility with legacy Flink Table API. */ - @Deprecated(since = "2.1.0", forRemoval = false) @Override - public DataStream> toRetractStream( - Table table, TypeInformation typeInformation) { - isConvertedToDataStream = true; - return getStreamTableEnv().toRetractStream(table, typeInformation); + public void createTemporaryModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { + getStreamTableEnv().createTemporaryModel(path, descriptor, ignoreIfExists); } @Override - public DataStream toAppendStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return getStreamTableEnv().toAppendStream(table, clazz); + public Table fromCall(String path, Object... arguments) { + return getStreamTableEnv().fromCall(path, arguments); } @Override - public DataStream> toRetractStream(Table table, Class clazz) { - isConvertedToDataStream = true; - return getStreamTableEnv().toRetractStream(table, clazz); + public Table fromCall(Class function, Object... arguments) { + return getStreamTableEnv().fromCall(function, arguments); } @Override - public boolean createTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { - return getStreamTableEnv().createTable(path, descriptor, ignoreIfExists); + public Model fromModel(String modelPath) { + return getStreamTableEnv().fromModel(modelPath); } @Override - public void createTemporaryTable( - String path, TableDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); + public Model fromModel(ModelDescriptor descriptor) { + return getStreamTableEnv().fromModel(descriptor); } @Override - public boolean createView(String path, Table view, boolean ignoreIfExists) { - return getStreamTableEnv().createView(path, view, ignoreIfExists); + public String[] listMaterializedTables() { + return getStreamTableEnv().listMaterializedTables(); } @Override - public void createView(String path, Table view) { - getStreamTableEnv().createView(path, view); + public String[] listModels() { + return getStreamTableEnv().listModels(); } @Override - public boolean dropTable(String path, boolean ignoreIfNotExists) { - return getStreamTableEnv().dropTable(path, ignoreIfNotExists); + public String[] listTemporaryModels() { + return getStreamTableEnv().listTemporaryModels(); } @Override @@ -296,8 +140,8 @@ public boolean dropTable(String path) { } @Override - public boolean dropView(String path, boolean ignoreIfNotExists) { - return getStreamTableEnv().dropView(path, ignoreIfNotExists); + public boolean dropTable(String path, boolean ignoreIfNotExists) { + return getStreamTableEnv().dropTable(path, ignoreIfNotExists); } @Override @@ -306,24 +150,13 @@ public boolean dropView(String path) { } @Override - public void createModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createModel(path, descriptor, ignoreIfExists); - } - - @Override - public void createModel(String path, ModelDescriptor descriptor) { - getStreamTableEnv().createModel(path, descriptor); - } - - @Override - public void createTemporaryModel( - String path, ModelDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createTemporaryModel(path, descriptor, ignoreIfExists); + public boolean dropView(String path, boolean ignoreIfNotExists) { + return getStreamTableEnv().dropView(path, ignoreIfNotExists); } @Override - public void createTemporaryModel(String path, ModelDescriptor descriptor) { - getStreamTableEnv().createTemporaryModel(path, descriptor); + public boolean dropModel(String path) { + return getStreamTableEnv().dropModel(path); } @Override @@ -331,69 +164,21 @@ public boolean dropModel(String path, boolean ignoreIfNotExists) { return getStreamTableEnv().dropModel(path, ignoreIfNotExists); } - @Override - public boolean dropModel(String path) { - return getStreamTableEnv().dropModel(path); - } - @Override public boolean dropTemporaryModel(String path) { return getStreamTableEnv().dropTemporaryModel(path); } @Override - public Table fromCall(Class functionClass, Object... arguments) { - return getStreamTableEnv().fromCall(functionClass, arguments); - } - - @Override - public Table fromCall(String functionName, Object... arguments) { - return getStreamTableEnv().fromCall(functionName, arguments); - } - - @Override - public String[] listModels() { - return getStreamTableEnv().listModels(); - } - - @Override - public String[] listTemporaryModels() { - return getStreamTableEnv().listTemporaryModels(); - } - - @Override - public void createFunction( - String path, FunctionDescriptor descriptor, boolean ignoreIfExists) { - getStreamTableEnv().createFunction(path, descriptor, ignoreIfExists); - } - - @Override - public void createFunction(String path, FunctionDescriptor descriptor) { - getStreamTableEnv().createFunction(path, descriptor); - } - - @Override - public void createTemporaryFunction(String path, FunctionDescriptor descriptor) { - getStreamTableEnv().createTemporaryFunction(path, descriptor); - } - - @Override - public void createTemporarySystemFunction(String name, FunctionDescriptor descriptor) { - getStreamTableEnv().createTemporarySystemFunction(name, descriptor); - } - - @Override - public Model fromModel(ModelDescriptor descriptor) { - return getStreamTableEnv().fromModel(descriptor); + public DataStream toChangelogStream(Table table, Schema targetSchema, ChangelogMode changelogMode, + InsertConflictStrategy conflictStrategy) { + isConvertedToDataStream = true; + return getStreamTableEnv().toChangelogStream(table, targetSchema, changelogMode, conflictStrategy); } @Override - public Model fromModel(String path) { - return getStreamTableEnv().fromModel(path); + public StreamStatementSet createStatementSet() { + return getStreamTableEnv().createStatementSet(); } - @Override - public String[] listMaterializedTables() { - return getStreamTableEnv().listMaterializedTables(); - } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/TableContext.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/TableContext.java index 9861a3876e..2e46c912ee 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/TableContext.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/TableContext.java @@ -17,114 +17,51 @@ package org.apache.streampark.flink.core; -import org.apache.flink.table.api.CompiledPlan; -import org.apache.flink.table.api.ExplainDetail; -import org.apache.flink.table.api.ExplainFormat; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.core.bean.TableContextSpec; + import org.apache.flink.table.api.FunctionDescriptor; import org.apache.flink.table.api.Model; import org.apache.flink.table.api.ModelDescriptor; -import org.apache.flink.table.api.PlanReference; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableDescriptor; import org.apache.flink.table.api.TableEnvironment; -import org.apache.flink.table.api.TableException; -import org.apache.flink.table.catalog.CatalogDescriptor; import org.apache.flink.table.functions.UserDefinedFunction; -import org.apache.flink.table.module.ModuleEntry; -import org.apache.flink.table.resource.ResourceUri; -import org.apache.flink.util.ParameterTool; - -import java.util.List; /** Flink 2.3 table environment context. */ -public class TableContext extends FlinkTableTrait { +public class TableContext extends AbstractFlinkTable { - public TableContext(ParameterTool parameter, TableEnvironment tableEnv) { + public TableContext(FlinkJobParameters parameter, TableEnvironment tableEnv) { super(parameter, tableEnv); } - public TableContext(FlinkTableInitializerV2.TableInitResult init) { - this(init.parameter, init.tableEnv); - } - - public TableContext(TableEnvConfig config) { - this(FlinkTableInitializerV2.initialize(config)); - } - - @Override - public void useModules(String... moduleNames) { - getTableEnv().useModules(moduleNames); - } - - @Override - public void createTemporaryTable(String path, TableDescriptor descriptor) { - getTableEnv().createTemporaryTable(path, descriptor); - } - - @Override - public void createTable(String path, TableDescriptor descriptor) { - getTableEnv().createTable(path, descriptor); - } - - @Override - public Table from(TableDescriptor descriptor) { - return getTableEnv().from(descriptor); - } - - @Override - public ModuleEntry[] listFullModules() { - return getTableEnv().listFullModules(); - } - - @Override - public String[] listTables(String catalogName, String databaseName) { - return getTableEnv().listTables(catalogName, databaseName); - } - - @Override - public CompiledPlan loadPlan(PlanReference planReference) throws TableException { - return getTableEnv().loadPlan(planReference); - } - - @Override - public CompiledPlan compilePlanSql(String statement) throws TableException { - return getTableEnv().compilePlanSql(statement); - } - - @Override - public void createFunction(String path, String className, List resourceUris) { - getTableEnv().createFunction(path, className, resourceUris); + public TableContext(TableContextSpec contextConfig) { + this(contextConfig.parameter, contextConfig.tableEnv); } @Override - public void createFunction( - String path, - String className, - List resourceUris, - boolean ignoreIfExists) { - getTableEnv().createFunction(path, className, resourceUris, ignoreIfExists); + public void createFunction(String path, FunctionDescriptor functionDescriptor) { + getTableEnv().createFunction(path, functionDescriptor); } @Override - public void createTemporaryFunction( - String path, String className, List resourceUris) { - getTableEnv().createTemporaryFunction(path, className, resourceUris); + public void createFunction(String path, FunctionDescriptor functionDescriptor, boolean ignoreIfExists) { + getTableEnv().createFunction(path, functionDescriptor, ignoreIfExists); } @Override - public void createTemporarySystemFunction( - String name, String className, List resourceUris) { - getTableEnv().createTemporarySystemFunction(name, className, resourceUris); + public void createTemporaryFunction(String path, FunctionDescriptor functionDescriptor) { + getTableEnv().createTemporaryFunction(path, functionDescriptor); } @Override - public String explainSql(String statement, ExplainFormat format, ExplainDetail... extraDetails) { - return getTableEnv().explainSql(statement, format, extraDetails); + public void createTemporarySystemFunction(String name, FunctionDescriptor functionDescriptor) { + getTableEnv().createTemporarySystemFunction(name, functionDescriptor); } @Override - public void createCatalog(String catalogName, CatalogDescriptor catalogDescriptor) { - getTableEnv().createCatalog(catalogName, catalogDescriptor); + public void createTemporaryTable(String path, TableDescriptor descriptor, boolean ignoreIfExists) { + getTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); } @Override @@ -132,40 +69,19 @@ public boolean createTable(String path, TableDescriptor descriptor, boolean igno return getTableEnv().createTable(path, descriptor, ignoreIfExists); } - @Override - public void createTemporaryTable( - String path, TableDescriptor descriptor, boolean ignoreIfExists) { - getTableEnv().createTemporaryTable(path, descriptor, ignoreIfExists); - } - - @Override - public boolean createView(String path, Table view, boolean ignoreIfExists) { - return getTableEnv().createView(path, view, ignoreIfExists); - } - @Override public void createView(String path, Table view) { getTableEnv().createView(path, view); } @Override - public boolean dropTable(String path, boolean ignoreIfNotExists) { - return getTableEnv().dropTable(path, ignoreIfNotExists); - } - - @Override - public boolean dropTable(String path) { - return getTableEnv().dropTable(path); - } - - @Override - public boolean dropView(String path, boolean ignoreIfNotExists) { - return getTableEnv().dropView(path, ignoreIfNotExists); + public boolean createView(String path, Table view, boolean ignoreIfExists) { + return getTableEnv().createView(path, view, ignoreIfExists); } @Override - public boolean dropView(String path) { - return getTableEnv().dropView(path); + public void createModel(String path, ModelDescriptor descriptor) { + getTableEnv().createModel(path, descriptor); } @Override @@ -174,44 +90,38 @@ public void createModel(String path, ModelDescriptor descriptor, boolean ignoreI } @Override - public void createModel(String path, ModelDescriptor descriptor) { - getTableEnv().createModel(path, descriptor); + public void createTemporaryModel(String path, ModelDescriptor descriptor) { + getTableEnv().createTemporaryModel(path, descriptor); } @Override - public void createTemporaryModel( - String path, ModelDescriptor descriptor, boolean ignoreIfExists) { + public void createTemporaryModel(String path, ModelDescriptor descriptor, boolean ignoreIfExists) { getTableEnv().createTemporaryModel(path, descriptor, ignoreIfExists); } @Override - public void createTemporaryModel(String path, ModelDescriptor descriptor) { - getTableEnv().createTemporaryModel(path, descriptor); - } - - @Override - public boolean dropModel(String path, boolean ignoreIfNotExists) { - return getTableEnv().dropModel(path, ignoreIfNotExists); + public Table fromCall(String path, Object... arguments) { + return getTableEnv().fromCall(path, arguments); } @Override - public boolean dropModel(String path) { - return getTableEnv().dropModel(path); + public Table fromCall(Class function, Object... arguments) { + return getTableEnv().fromCall(function, arguments); } @Override - public boolean dropTemporaryModel(String path) { - return getTableEnv().dropTemporaryModel(path); + public Model fromModel(String modelPath) { + return getTableEnv().fromModel(modelPath); } @Override - public Table fromCall(Class functionClass, Object... arguments) { - return getTableEnv().fromCall(functionClass, arguments); + public Model fromModel(ModelDescriptor descriptor) { + return getTableEnv().fromModel(descriptor); } @Override - public Table fromCall(String functionName, Object... arguments) { - return getTableEnv().fromCall(functionName, arguments); + public String[] listMaterializedTables() { + return getTableEnv().listMaterializedTables(); } @Override @@ -225,38 +135,37 @@ public String[] listTemporaryModels() { } @Override - public void createFunction( - String path, FunctionDescriptor descriptor, boolean ignoreIfExists) { - getTableEnv().createFunction(path, descriptor, ignoreIfExists); + public boolean dropTable(String path) { + return getTableEnv().dropTable(path); } @Override - public void createFunction(String path, FunctionDescriptor descriptor) { - getTableEnv().createFunction(path, descriptor); + public boolean dropTable(String path, boolean ignoreIfNotExists) { + return getTableEnv().dropTable(path, ignoreIfNotExists); } @Override - public void createTemporaryFunction(String path, FunctionDescriptor descriptor) { - getTableEnv().createTemporaryFunction(path, descriptor); + public boolean dropView(String path) { + return getTableEnv().dropView(path); } @Override - public void createTemporarySystemFunction(String name, FunctionDescriptor descriptor) { - getTableEnv().createTemporarySystemFunction(name, descriptor); + public boolean dropView(String path, boolean ignoreIfNotExists) { + return getTableEnv().dropView(path, ignoreIfNotExists); } @Override - public Model fromModel(ModelDescriptor descriptor) { - return getTableEnv().fromModel(descriptor); + public boolean dropModel(String path) { + return getTableEnv().dropModel(path); } @Override - public Model fromModel(String path) { - return getTableEnv().fromModel(path); + public boolean dropModel(String path, boolean ignoreIfNotExists) { + return getTableEnv().dropModel(path, ignoreIfNotExists); } @Override - public String[] listMaterializedTables() { - return getTableEnv().listMaterializedTables(); + public boolean dropTemporaryModel(String path) { + return getTableEnv().dropTemporaryModel(path); } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/TableExt.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/TableExt.java deleted file mode 100644 index be1daab533..0000000000 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims_flink-2.3/src/main/java/org/apache/streampark/flink/core/TableExt.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.core; - -import org.apache.flink.streaming.api.datastream.DataStream; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.types.Row; - -/** Table API extensions for Flink 2.3 Java stream-table applications. */ -public final class TableExt { - - private TableExt() { - } - - /** Table alias helper (Scala {@code ->} operator equivalent: {@code as}). */ - public static final class Table { - - private final org.apache.flink.table.api.Table flinkTable; - - public Table(org.apache.flink.table.api.Table table) { - this.flinkTable = table; - } - - public org.apache.flink.table.api.Table as(String field, String... fields) { - return flinkTable.as(field, fields); - } - } - - /** Table-to-DataStream conversion helpers. */ - public static class TableConversions { - - private final org.apache.flink.table.api.Table flinkTable; - - private final StreamTableEnvironment streamTableEnv; - - public TableConversions( - org.apache.flink.table.api.Table table, StreamTableEnvironment streamTableEnv) { - this.flinkTable = table; - this.streamTableEnv = streamTableEnv; - } - - /** Changelog stream conversion (Scala {@code \\} operator equivalent). */ - public DataStream toChangelogDataStream() { - return streamTableEnv.toDataStream(flinkTable); - } - } -} diff --git a/streampark-flink/streampark-flink-sqlclient/src/main/java/org/apache/streampark/flink/cli/SqlClient.java b/streampark-flink/streampark-flink-sqlclient/src/main/java/org/apache/streampark/flink/cli/SqlClient.java index 88d957db54..f7b28b6d38 100644 --- a/streampark-flink/streampark-flink-sqlclient/src/main/java/org/apache/streampark/flink/cli/SqlClient.java +++ b/streampark-flink/streampark-flink-sqlclient/src/main/java/org/apache/streampark/flink/cli/SqlClient.java @@ -17,27 +17,32 @@ package org.apache.streampark.flink.cli; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.CommandLineParser; +import org.apache.streampark.common.configuration.Configuration; +import org.apache.streampark.common.configuration.ConfigurationFormat; +import org.apache.streampark.common.configuration.ConfigurationParser; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.option.ApplicationOptions; +import org.apache.streampark.common.configuration.option.CoreOptions; import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.PropertiesUtils; import org.apache.streampark.common.util.SystemPropertyUtils; +import org.apache.streampark.flink.configuration.FlinkJobParameters; +import org.apache.streampark.flink.core.FlinkTableInitializer; import org.apache.streampark.flink.core.SqlCommand; import org.apache.streampark.flink.core.SqlCommandCall; import org.apache.streampark.flink.core.SqlCommandParser; import org.apache.streampark.flink.core.StreamTableContext; -import org.apache.streampark.flink.core.StreamTableEnvConfig; import org.apache.streampark.flink.core.TableContext; -import org.apache.streampark.flink.core.TableEnvConfig; +import org.apache.streampark.flink.core.bean.StreamTableContextSpec; +import org.apache.streampark.flink.core.bean.TableContextSpec; import org.apache.commons.lang3.StringUtils; import org.apache.flink.api.common.RuntimeExecutionMode; -import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.configuration.ExecutionOptions; import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Map; /** Flink SQL job CLI entry point. */ public final class SqlClient { @@ -48,10 +53,9 @@ private SqlClient() { public static void main(String[] args) { List arguments = new ArrayList<>(Arrays.asList(args)); - ParameterTool parameterTool = ParameterTool.fromArgs(args); + FlinkJobParameters parameters = FlinkJobParameters.of(CommandLineParser.parse(args)); - String sqlKey = ConfigKeys.KEY_FLINK_SQL(); - String sql = parameterTool.get(sqlKey); + String sql = parameters.get(ApplicationOptions.SQL); if (StringUtils.isBlank(sql)) { throw new IllegalArgumentException("Usage: flink sql cannot be null"); } @@ -68,8 +72,7 @@ public static void main(String[] args) { } String defaultMode = RuntimeExecutionMode.STREAMING.name(); - String mode = - resolveExecutionMode(parameterTool, sets, arguments, defaultMode); + String mode = resolveExecutionMode(parameters, sets, arguments, defaultMode); switch (mode) { case "STREAMING": @@ -86,7 +89,7 @@ public static void main(String[] args) { } static String resolveExecutionMode( - ParameterTool parameterTool, + FlinkJobParameters parameters, List sets, List arguments, String defaultMode) { @@ -94,41 +97,71 @@ static String resolveExecutionMode( if (setCall.operands.length >= 2 && ExecutionOptions.RUNTIME_MODE.key().equals(setCall.operands[0])) { String runtimeMode = setCall.operands[1].toUpperCase(); - arguments.add("-D" + ExecutionOptions.RUNTIME_MODE.key() + "=" + runtimeMode); + setArgument(arguments, ExecutionOptions.RUNTIME_MODE.key(), runtimeMode); return runtimeMode; } } - String configuredMode = parameterTool.get(ExecutionOptions.RUNTIME_MODE.key(), null); + String configuredMode = parameters.get(ExecutionOptions.RUNTIME_MODE.key(), null); if (configuredMode != null) { return configuredMode; } - String appConf = parameterTool.get(ConfigKeys.KEY_APP_CONF(), null); + String appConf = parameters.getOptional(FlinkOptions.APPLICATION_CONFIG).orElse(null); String runtimeMode; if (appConf == null) { runtimeMode = defaultMode; } else { - Map parameter = - PropertiesUtils.fromYamlText(DeflaterUtils.unzipString(appConf.substring(7))); + Configuration applicationConfiguration = parseApplicationConfiguration(appConf); runtimeMode = - parameter - .getOrDefault(ConfigKeys.KEY_FLINK_TABLE_MODE(), defaultMode) + applicationConfiguration + .getOptional(FlinkOptions.TABLE_MODE) + .orElse(defaultMode) .toUpperCase(); } - arguments.add("-D" + ExecutionOptions.RUNTIME_MODE.key() + "=" + runtimeMode); + setArgument(arguments, ExecutionOptions.RUNTIME_MODE.key(), runtimeMode); return runtimeMode; } + private static Configuration parseApplicationConfiguration(String resource) { + if (resource.startsWith("yaml://")) { + return ConfigurationParser.parse( + DeflaterUtils.unzipString(resource.substring("yaml://".length())), + ConfigurationFormat.YAML, + "inline SQL application configuration"); + } + return ConfigurationParser.parse(java.nio.file.Path.of(resource)); + } + + private static void setArgument(List arguments, String key, String value) { + String option = CommandLineParser.LONG_OPTION_PREFIX + key; + for (int index = 0; index < arguments.size(); index++) { + if (option.equals(arguments.get(index))) { + if (index + 1 < arguments.size()) { + arguments.set(index + 1, value); + return; + } + break; + } + if (arguments.get(index).startsWith(option + "=")) { + arguments.set(index, option + "=" + value); + return; + } + } + arguments.add(option); + arguments.add(value); + } + private static final class BatchSqlApp { private BatchSqlApp() { } static void run(String[] args) { - SystemPropertyUtils.setAppHome(ConfigKeys.KEY_APP_HOME(), SqlClient.class); - TableContext context = new TableContext(new TableEnvConfig(args, null)); - context.sql(null); + SystemPropertyUtils.setAppHome(CoreOptions.APP_HOME.key(), SqlClient.class); + TableContextSpec contextConfig = FlinkTableInitializer.initializeTable(args); + TableContext context = new TableContext(contextConfig); + context.sql(); context.start(); } } @@ -139,10 +172,10 @@ private StreamSqlApp() { } static void run(String[] args) { - SystemPropertyUtils.setAppHome(ConfigKeys.KEY_APP_HOME(), SqlClient.class); - StreamTableContext context = - new StreamTableContext(new StreamTableEnvConfig(args, null, null)); - context.sql(null); + SystemPropertyUtils.setAppHome(CoreOptions.APP_HOME.key(), SqlClient.class); + StreamTableContextSpec contextConfig = FlinkTableInitializer.initializeStreamTable(args); + StreamTableContext context = new StreamTableContext(contextConfig); + context.sql(); context.start(); } } diff --git a/streampark-flink/streampark-flink-sqlclient/src/test/java/org/apache/streampark/flink/cli/SqlClientTest.java b/streampark-flink/streampark-flink-sqlclient/src/test/java/org/apache/streampark/flink/cli/SqlClientTest.java index b69ae36faf..9ea00d3910 100644 --- a/streampark-flink/streampark-flink-sqlclient/src/test/java/org/apache/streampark/flink/cli/SqlClientTest.java +++ b/streampark-flink/streampark-flink-sqlclient/src/test/java/org/apache/streampark/flink/cli/SqlClientTest.java @@ -17,21 +17,22 @@ package org.apache.streampark.flink.cli; -import org.apache.streampark.common.conf.ConfigKeys; +import org.apache.streampark.common.configuration.CommandLineParser; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.option.ApplicationOptions; +import org.apache.streampark.common.util.DeflaterUtils; +import org.apache.streampark.flink.configuration.FlinkJobParameters; import org.apache.streampark.flink.core.SqlCommand; import org.apache.streampark.flink.core.SqlCommandCall; import org.apache.flink.api.common.RuntimeExecutionMode; -import org.apache.flink.api.java.utils.ParameterTool; import org.apache.flink.configuration.ExecutionOptions; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -52,27 +53,27 @@ void resolveExecutionModeFromSetCommand() { String mode = SqlClient.resolveExecutionMode( - ParameterTool.fromArgs(new String[0]), + FlinkJobParameters.of(CommandLineParser.parse(new String[0])), sets, arguments, RuntimeExecutionMode.STREAMING.name()); assertEquals("BATCH", mode); - assertTrue( - arguments.contains("-D" + ExecutionOptions.RUNTIME_MODE.key() + "=BATCH")); + assertTrue(arguments.contains("--" + ExecutionOptions.RUNTIME_MODE.key())); + assertTrue(arguments.contains("BATCH")); } @Test void resolveExecutionModeFromDynamicProperty() { List arguments = new ArrayList<>(); - Map properties = new HashMap<>(); - properties.put(ExecutionOptions.RUNTIME_MODE.key(), "BATCH"); - ParameterTool parameterTool = - ParameterTool.fromSystemProperties().mergeWith(ParameterTool.fromMap(properties)); + FlinkJobParameters parameters = + FlinkJobParameters.of( + CommandLineParser.parse( + new String[]{"--" + ExecutionOptions.RUNTIME_MODE.key(), "BATCH"})); String mode = SqlClient.resolveExecutionMode( - parameterTool, + parameters, Collections.emptyList(), arguments, RuntimeExecutionMode.STREAMING.name()); @@ -84,21 +85,22 @@ void resolveExecutionModeFromDynamicProperty() { @Test void resolveExecutionModeFromAppConfYaml() { List arguments = new ArrayList<>(); - String yamlContent = ConfigKeys.KEY_FLINK_TABLE_MODE() + ": batch\n"; - String appConf = "yaml://" + org.apache.streampark.common.util.DeflaterUtils.zipString(yamlContent); - Map params = new HashMap<>(); - params.put(ConfigKeys.KEY_APP_CONF(), appConf); - ParameterTool parameterTool = ParameterTool.fromMap(params); + String yamlContent = FlinkOptions.TABLE_MODE.key() + ": batch\n"; + String appConf = "yaml://" + DeflaterUtils.zipString(yamlContent); + FlinkJobParameters parameters = + FlinkJobParameters.of( + CommandLineParser.parse( + new String[]{"--" + ApplicationOptions.CONFIG.key(), appConf})); String mode = SqlClient.resolveExecutionMode( - parameterTool, + parameters, Collections.emptyList(), arguments, RuntimeExecutionMode.STREAMING.name()); assertEquals("BATCH", mode); - assertTrue( - arguments.contains("-D" + ExecutionOptions.RUNTIME_MODE.key() + "=BATCH")); + assertTrue(arguments.contains("--" + ExecutionOptions.RUNTIME_MODE.key())); + assertTrue(arguments.contains("BATCH")); } } diff --git a/streampark-flink/streampark-flink-udf/pom.xml b/streampark-flink/streampark-flink-udf/pom.xml deleted file mode 100644 index 7c46cabfb8..0000000000 --- a/streampark-flink/streampark-flink-udf/pom.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - 4.0.0 - - org.apache.streampark - streampark-flink - ${revision} - - - streampark-flink-udf - StreamPark : Flink Udf - - - - org.apache.streampark - streampark-common - provided - - - commons-cli - commons-cli - - - redis.clients - jedis - - - - - - org.apache.flink - flink-table-common - ${flink.version} - provided - - - - - - - - - - - apache-release - - true - - - - - diff --git a/streampark-flink/streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/DateFormat.java b/streampark-flink/streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/DateFormat.java deleted file mode 100644 index e7d9bda8c6..0000000000 --- a/streampark-flink/streampark-flink-udf/src/main/java/org/apache/streampark/flink/udf/DateFormat.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.udf; - -import org.apache.flink.table.functions.ScalarFunction; - -import java.text.SimpleDateFormat; -import java.util.Date; - -public class DateFormat extends ScalarFunction { - - private final SimpleDateFormat simpleDateFormat; - - public DateFormat() { - this.simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - } - - public DateFormat(String format) { - this.simpleDateFormat = new SimpleDateFormat(format); - } - - public String eval(Long time) { - return simpleDateFormat.format(new Date(time)); - } -} diff --git a/streampark-common-scala-bridge/pom.xml b/streampark-scala/pom.xml similarity index 72% rename from streampark-common-scala-bridge/pom.xml rename to streampark-scala/pom.xml index 2344333ba3..4339753105 100644 --- a/streampark-common-scala-bridge/pom.xml +++ b/streampark-scala/pom.xml @@ -24,10 +24,28 @@ ${revision} - streampark-common-scala-bridge_${scala.binary.version} - StreamPark : Common Scala Bridge + streampark-scala_${scala.binary.version} + StreamPark : Scala + + org.scala-lang + scala-library + provided + + + + org.scala-lang + scala-compiler + provided + + + + org.scala-lang + scala-reflect + provided + + org.apache.streampark streampark-common diff --git a/streampark-common-scala-bridge/src/main/scala/org/apache/streampark/common/util/Implicits.scala b/streampark-scala/src/main/scala/org/apache/streampark/common/util/Implicits.scala similarity index 100% rename from streampark-common-scala-bridge/src/main/scala/org/apache/streampark/common/util/Implicits.scala rename to streampark-scala/src/main/scala/org/apache/streampark/common/util/Implicits.scala diff --git a/streampark-common-scala-bridge/src/main/scala/org/apache/streampark/common/util/Logger.scala b/streampark-scala/src/main/scala/org/apache/streampark/common/util/Logger.scala similarity index 100% rename from streampark-common-scala-bridge/src/main/scala/org/apache/streampark/common/util/Logger.scala rename to streampark-scala/src/main/scala/org/apache/streampark/common/util/Logger.scala diff --git a/streampark-spark/pom.xml b/streampark-spark/pom.xml index 61990f5196..ceb0b58781 100644 --- a/streampark-spark/pom.xml +++ b/streampark-spark/pom.xml @@ -44,7 +44,7 @@ org.apache.streampark - streampark-common-scala-bridge_${scala.binary.version} + streampark-scala_${scala.binary.version} @@ -61,6 +61,10 @@ org.slf4j slf4j-log4j12 + + org.apache.hadoop + hadoop-client + @@ -68,33 +72,12 @@ org.apache.spark spark-sql_2.12 ${spark.version} - - - - org.apache.spark - spark-streaming-kafka-0-10_2.12 - ${spark.version} - - - - com.esotericsoftware - kryo-shaded - 4.0.2 - provided - - - - org.json4s - json4s-ast_2.12 - ${json4s-jackson.version} - provided - - - - org.json4s - json4s-core_2.12 - ${json4s-jackson.version} - provided + + + org.apache.hadoop + hadoop-client + + @@ -108,13 +91,6 @@ hadoop-client-runtime provided - - - org.apache.hbase - hbase-hadoop2-compat - ${hbase.version} - provided - diff --git a/streampark-spark/streampark-spark-cli/create_app.sh b/streampark-spark/streampark-spark-cli/create_app.sh deleted file mode 100644 index 998811ee5d..0000000000 --- a/streampark-spark/streampark-spark-cli/create_app.sh +++ /dev/null @@ -1,412 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -base=$(cd $(dirname $0);pwd) - - -__cmd__() { - if which ${1:-"cmd"} >/dev/null 2>&1;then echo 1;else echo 0;fi -} - -__get__() { - local i=$1 - test "x$i" == "x" && read -p "Please enter $2: " -t 30 i - test "x$i" == "x" && { echo "0x0F";return; } - echo $i -} -__exit__() { - test "x$1" == "x0x0F" && { echo "parameter error.">&2;exit; } -} - -create_demo_code(){ - local gourp_id=$1 - local module=$2 - local code_dir=$3 - -cat > $code_dir/Demo.scala < (x.topic,x.value)).foreachRDD((rdd,time) => { - rdd.take(10).foreach(println) - source.updateOffsets(time.milliseconds) - }) - } -} -EOF -} - -create_module() { - local module=$(__get__ "$1" "module name") - __exit__ "$module" - local src_dir=src/main/scala - local code_dir=${src_dir}/${group_id//./\/}/${module} - local deploy_dir=assembly - local conf_dir=${deploy_dir}/conf/prod - local log_dir=${deploy_dir}/logs - local temp_dir=${deploy_dir}/temp - - mkdir -p ${name}/${module} - test ! -d ${name}/${module} && { echo "create modulue dir failed.">&2;exit; } - - mkdir -p ${name}/${module}/${code_dir} - mkdir -p $name/${module}/${src_dir/main/test} - mkdir -p ${name}/${module}/${conf_dir} - mkdir -p ${name}/${module}/${log_dir} - mkdir -p ${name}/${module}/${temp_dir} - - cp -r ${base}/bin ${name}/${module}/assembly/bin - cp ${name}/default.properties ${name}/${module}/${conf_dir}/${module}.properties - - echo "# $module module documentation" >${name}/${module}/README.md - cat > ${name}/${module}/assembly.xml < - bin - - tar.gz - - - - true - lib - true - - org.apache.streampark:streampark-spark - - - - - false - lib - true - provided - - - - - / - - README.md - - - - \${project.basedir}/assembly - / - - - - \${project.build.directory} - lib - - *.jar - - - - - - \${project.build.directory} - lib - - *.jar - - - - assembly/bin - bin - 0755 - - - assembly/conf - conf - 0755 - - - assembly/logs - logs - 0755 - - - assembly/logs - temp - 0755 - - - -EOF - - cat >$name/$module/pom.xml < - - - $artifact_id - $group_id - $version - ../pom.xml - - 4.0.0 - - $module - - - - org.apache.streampark - streampark-spark - - - - - - - - maven-assembly-plugin - - - - - - -EOF -create_demo_code $group_id $module $name/$module/$code_dir -} - -create_pom() { - local group_id=$(__get__ "$1" "group id") - __exit__ "$group_id" - local artifact_id=$(__get__ "$2" "artifact id") - __exit__ "$artifact_id" - local module=$(__get__ "$3" "module name") - __exit__ "$module" - local version=$(__get__ "${4:-"1.0"}" "version") - __exit__ "$version" - -cat > $name/pom.xml < - - 4.0.0 - - $group_id - $artifact_id - pom - $version - - - - - default - - compile - - - true - - - - - withjar - - provided - - - - - $module - - - - - 1.0.0 - - 2.2.0 - 2.10.1 - 1.2.0-cdh5.12.1 - 1.1.0-cdh5.12.1 - - 2.11.12 - 2.11 - 2.8.2 - 5.1.6 - 0.10.2.0 - 5.6.3 - 2.5.0 - - 3.2.10 - 1.3.3 - 2.3.9 - - UTF-8 - UTF-8 - 11 - - - 64m - 512m - 512m - - - - - - - - org.apache.streampark - streampark-spark-core - \${streampark.spark.version} - - - - - - - src/main/scala - - - - src/main/resources - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - \${project.build.jdk} - \${project.build.jdk} - - - - - net.alchim31.maven - scala-maven-plugin - 4.3.0 - - - compile-scala - compile - - add-source - compile - - - - test-compile-scala - test-compile - - testCompile - - - - - \${scala.version} - - - - - - - - maven-assembly-plugin - 3.0.0 - - - distro-assembly - package - - single - - - - - false - - assembly.xml - - - - - - - - - -EOF - create_module "$module" -} - -create_git_filter() { - echo -e "# Createdby .ignore support plugin (hsz.mobi)\n.idea/\n*.iml\ntarget/\n.DS_Store" >$name/.gitignore -} - -create_readme() { - echo -e "# $name project instruction" >$name/README.md -} - -create_default_conf() { - cp $base/create_conf.sh $name - bash $base/create_conf.sh > $name/default.properties -} - -create_project() { - local name=$1 - test "x$name" == "x" && read -p "Please enter name: " -t 30 name - test "x$name" == "x" && { echo "The name of the project can not be empty" >&2;exit; } - mkdir -p $name - test ! -d $name && { echo "project create failed." >&2;exit; } - create_git_filter - create_readme - create_default_conf - create_pom "$2" "$3" "$4" "$5" - echo "procject $name create successful." -} -test $# -eq 0 && { - echo -e "Usage:\n\tbash $0 [procject_name]\nEx:\n\tbash $0 ~/code/test_project" >&2; - exit; -} -create_project "$@" diff --git a/streampark-spark/streampark-spark-cli/create_conf.sh b/streampark-spark/streampark-spark-cli/create_conf.sh deleted file mode 100644 index f2148186d0..0000000000 --- a/streampark-spark/streampark-spark-cli/create_conf.sh +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -test -f /etc/profile && . /etc/profile -test -f $HOME/.bash_profile && . $HOME/.bash_profile -base=$(cd $(dirname $0);pwd) - -function __check__(){ - local cmd=${1:-"gawk"} - if ! which $cmd >/dev/null 2>&1;then - echo "Please install $cmd" >&2 - exit - fi -} -function create_notes(){ - local msg=$1 - local col=${2:-55} - __check__ - gawk -vc=$col 'function as(ll,is,ls){for(i=1;i - - - 4.0.0 - - org.apache.streampark - streampark-spark - ${revision} - - - streampark-spark-cli - StreamPark : Spark Cli - pom - - diff --git a/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/java/org/apache/streampark/spark/configuration/SparkOptions.java b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/java/org/apache/streampark/spark/configuration/SparkOptions.java new file mode 100644 index 0000000000..23f163635b --- /dev/null +++ b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/java/org/apache/streampark/spark/configuration/SparkOptions.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.spark.configuration; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; +import org.apache.streampark.common.configuration.ConfigPrefix; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * Typed option catalog for Spark applications and Spark submission metadata. + * + *

Native Spark keys remain unchanged. Submission-only fields are declared here as explicit + * transport metadata so that callers do not confuse them with Spark's own configuration. + */ +public final class SparkOptions { + + public static final ConfigOption DRIVER_CORES = positiveInteger("spark.driver.cores"); + public static final ConfigOption DRIVER_MEMORY = string("spark.driver.memory"); + public static final ConfigOption EXECUTOR_INSTANCES = + positiveInteger("spark.executor.instances"); + public static final ConfigOption EXECUTOR_CORES = + positiveInteger("spark.executor.cores"); + public static final ConfigOption EXECUTOR_MEMORY = string("spark.executor.memory"); + public static final ConfigOption DYNAMIC_ALLOCATION_ENABLED = + ConfigOptions.key("spark.dynamicAllocation.enabled") + .booleanType() + .defaultValue(false) + .withDescription("Whether Spark dynamic resource allocation is enabled.") + .build(); + public static final ConfigOption DYNAMIC_ALLOCATION_MAX_EXECUTORS = + string("spark.dynamicAllocation.maxExecutors"); + public static final ConfigOption YARN_QUEUE = string("spark.yarn.queue"); + public static final ConfigOption YARN_QUEUE_NAME = string("yarnQueueName"); + public static final ConfigOption YARN_QUEUE_LABEL = string("yarnQueueLabel"); + public static final ConfigOption YARN_AM_NODE_LABEL = + string("spark.yarn.am.nodeLabelExpression"); + public static final ConfigOption YARN_EXECUTOR_NODE_LABEL = + string("spark.yarn.executor.nodeLabelExpression"); + + public static final ConfigPrefix PROPERTY_PREFIX = ConfigPrefix.of("spark."); + + public static final List> ALL = + Collections.unmodifiableList( + Arrays.asList( + DRIVER_CORES, + DRIVER_MEMORY, + EXECUTOR_INSTANCES, + EXECUTOR_CORES, + EXECUTOR_MEMORY, + DYNAMIC_ALLOCATION_ENABLED, + DYNAMIC_ALLOCATION_MAX_EXECUTORS, + YARN_QUEUE, + YARN_QUEUE_NAME, + YARN_QUEUE_LABEL, + YARN_AM_NODE_LABEL, + YARN_EXECUTOR_NODE_LABEL)); + + private SparkOptions() { + } + + private static ConfigOption string(String key) { + return ConfigOptions.key(key) + .stringType() + .noDefaultValue() + .withDescription("Spark option '" + key + "'.") + .build(); + } + + private static ConfigOption positiveInteger(String key) { + return ConfigOptions.key(key) + .intType() + .noDefaultValue() + .check(value -> value > 0, key + " must be greater than zero") + .withDescription("Spark option '" + key + "'.") + .build(); + } +} diff --git a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/entity/UserManagementUserType.java b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/java/org/apache/streampark/spark/configuration/package-info.java similarity index 87% rename from streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/entity/UserManagementUserType.java rename to streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/java/org/apache/streampark/spark/configuration/package-info.java index 34154068d0..7068fb52bd 100644 --- a/streampark-e2e/streampark-e2e-case/src/test/java/org/apache/streampark/e2e/pages/system/entity/UserManagementUserType.java +++ b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/java/org/apache/streampark/spark/configuration/package-info.java @@ -15,8 +15,5 @@ * limitations under the License. */ -package org.apache.streampark.e2e.pages.system.entity; - -public enum UserManagementUserType { - ADMIN, USER -} +/** Spark-specific typed configuration definitions. */ +package org.apache.streampark.spark.configuration; diff --git a/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/SparkClient.scala b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/SparkClient.scala index 2e198d075b..7f95646169 100644 --- a/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/SparkClient.scala +++ b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/SparkClient.scala @@ -17,7 +17,7 @@ package org.apache.streampark.spark.client -import org.apache.streampark.common.conf.SparkVersion +import org.apache.streampark.common.core.SparkVersion import org.apache.streampark.common.util.Logger import org.apache.streampark.spark.client.bean._ import org.apache.streampark.spark.client.proxy.SparkShimsProxy diff --git a/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/bean/CancelRequest.scala b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/bean/CancelRequest.scala index 22a2b4e08b..5a1bc4681f 100644 --- a/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/bean/CancelRequest.scala +++ b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/bean/CancelRequest.scala @@ -17,7 +17,7 @@ package org.apache.streampark.spark.client.bean -import org.apache.streampark.common.conf.SparkVersion +import org.apache.streampark.common.core.SparkVersion import org.apache.streampark.common.enums.SparkDeployMode import org.apache.streampark.common.util.Implicits.JavaMap diff --git a/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/bean/SubmitRequest.scala b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/bean/SubmitRequest.scala index 9161473fa4..fcc02efb8e 100644 --- a/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/bean/SubmitRequest.scala +++ b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/bean/SubmitRequest.scala @@ -17,13 +17,14 @@ package org.apache.streampark.spark.client.bean -import org.apache.streampark.common.conf.{SparkVersion, Workspace} -import org.apache.streampark.common.conf.ConfigKeys._ -import org.apache.streampark.common.constants.Constants +import org.apache.streampark.common.configuration.{ConfigurationFormat, ConfigurationParser, Constants} +import org.apache.streampark.common.configuration.Workspace +import org.apache.streampark.common.core.SparkVersion import org.apache.streampark.common.enums._ -import org.apache.streampark.common.util.{DeflaterUtils, HdfsUtils, PropertiesUtils} +import org.apache.streampark.common.util.{DeflaterUtils, HdfsUtils} import org.apache.streampark.common.util.Implicits._ import org.apache.streampark.flink.packer.pipeline.{BuildResult, ShadedBuildResponse} +import org.apache.streampark.spark.configuration.SparkOptions import com.fasterxml.jackson.databind.ObjectMapper import org.apache.commons.collections.MapUtils @@ -57,7 +58,7 @@ case class SubmitRequest( "spark.executor.instances" -> "2") lazy val sparkParameterMap: Map[String, String] = getParameterMap( - KEY_SPARK_PROPERTY_PREFIX) + SparkOptions.PROPERTY_PREFIX.value) lazy val appMain: String = this.jobType match { case SparkJobType.SPARK_SQL => Constants.STREAMPARK_SPARKSQL_CLIENT_CLASS @@ -88,9 +89,9 @@ case class SubmitRequest( } else { lazy val content = DeflaterUtils.unzipString(this.appConf.trim.drop(7)) val map = format match { - case "yaml://" => PropertiesUtils.fromYamlText(content) - case "conf://" => PropertiesUtils.fromHoconText(content) - case "prop://" => PropertiesUtils.fromPropertiesText(content) + case "yaml://" => parseConfig(content, ConfigurationFormat.YAML, "inline YAML") + case "conf://" => parseConfig(content, ConfigurationFormat.HOCON, "inline HOCON") + case "prop://" => parseConfig(content, ConfigurationFormat.PROPERTIES, "inline properties") case "hdfs://" => /** * If the configuration file is HDFS mode, you need to copy the HDFS related configuration @@ -99,9 +100,9 @@ case class SubmitRequest( val text = HdfsUtils.read(this.appConf) val extension = this.appConf.split("\\.").last.toLowerCase extension match { - case "yml" | "yaml" => PropertiesUtils.fromYamlText(text) - case "conf" => PropertiesUtils.fromHoconText(text) - case "properties" => PropertiesUtils.fromPropertiesText(text) + case "yml" | "yaml" => parseConfig(text, ConfigurationFormat.YAML, this.appConf) + case "conf" => parseConfig(text, ConfigurationFormat.HOCON, this.appConf) + case "properties" => parseConfig(text, ConfigurationFormat.PROPERTIES, this.appConf) case _ => throw new IllegalArgumentException( "[StreamPark] Usage: application config format error,must be [yaml|conf|properties]") @@ -115,6 +116,12 @@ case class SubmitRequest( } } + private def parseConfig( + content: String, + format: ConfigurationFormat, + origin: String): Map[String, String] = + ConfigurationParser.parse(content, format, origin).toMap.asScala.toMap + @throws[IOException] private def isSymlink(file: File): Boolean = { if (file == null) throw new NullPointerException("File must not be null") @@ -127,7 +134,7 @@ case class SubmitRequest( * The spark version and configuration in the native spark and hdfs must be kept exactly the * same. */ - val workspace = Workspace.remote + val workspace = Workspace.REMOTE val sparkHome = sparkVersion.sparkHome val sparkHomeDir = new File(sparkHome) val sparkName = if (isSymlink(sparkHomeDir)) { @@ -135,13 +142,13 @@ case class SubmitRequest( } else { sparkHomeDir.getName } - val sparkHdfsHome = s"${workspace.APP_SPARK}/$sparkName" + val sparkHdfsHome = s"${workspace.spark}/$sparkName" HdfsWorkspace( sparkName, sparkHome, sparkLib = s"$sparkHdfsHome/jars", sparkPlugins = s"$sparkHdfsHome/plugins", - appJars = workspace.APP_JARS) + appJars = workspace.jars) } @throws[Exception] diff --git a/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/proxy/SparkShimsProxy.scala b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/proxy/SparkShimsProxy.scala index f0a7933f04..5fbd737c4b 100644 --- a/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/proxy/SparkShimsProxy.scala +++ b/streampark-spark/streampark-spark-client/streampark-spark-client-api/src/main/scala/org/apache/streampark/spark/client/proxy/SparkShimsProxy.scala @@ -17,8 +17,9 @@ package org.apache.streampark.spark.client.proxy -import org.apache.streampark.common.conf.{ConfigKeys, SparkVersion} -import org.apache.streampark.common.constants.Constants +import org.apache.streampark.common.configuration.Constants +import org.apache.streampark.common.configuration.option.CoreOptions +import org.apache.streampark.common.core.SparkVersion import org.apache.streampark.common.util.{ChildFirstClassLoader, ClassLoaderObjectInputStream, ClassLoaderUtils, Logger} import org.apache.streampark.common.util.Implicits._ @@ -102,10 +103,10 @@ object SparkShimsProxy extends Logger { } private def addShimsUrls(sparkVersion: SparkVersion, addShimUrl: File => Unit): Unit = { - val appHome = System.getProperty(ConfigKeys.KEY_APP_HOME) + val appHome = System.getProperty(CoreOptions.APP_HOME.key) require( appHome != null, - String.format("%s is not found on System env.", ConfigKeys.KEY_APP_HOME)) + String.format("%s is not found on System env.", CoreOptions.APP_HOME.key)) val libPath = new File(s"$appHome/lib") require(libPath.exists()) diff --git a/streampark-spark/streampark-spark-client/streampark-spark-client-core/src/main/scala/org/apache/streampark/spark/client/impl/YarnClient.scala b/streampark-spark/streampark-spark-client/streampark-spark-client-core/src/main/scala/org/apache/streampark/spark/client/impl/YarnClient.scala index 007b80d06c..7a83b524e2 100644 --- a/streampark-spark/streampark-spark-client/streampark-spark-client-core/src/main/scala/org/apache/streampark/spark/client/impl/YarnClient.scala +++ b/streampark-spark/streampark-spark-client/streampark-spark-client-core/src/main/scala/org/apache/streampark/spark/client/impl/YarnClient.scala @@ -17,12 +17,12 @@ package org.apache.streampark.spark.client.impl -import org.apache.streampark.common.conf.ConfigKeys._ import org.apache.streampark.common.enums.SparkDeployMode import org.apache.streampark.common.util.{HadoopUtils, YarnUtils} import org.apache.streampark.common.util.Implicits._ import org.apache.streampark.spark.client.`trait`.SparkClientTrait import org.apache.streampark.spark.client.bean._ +import org.apache.streampark.spark.configuration.SparkOptions import org.apache.commons.lang3.StringUtils import org.apache.hadoop.yarn.api.records.ApplicationId @@ -166,12 +166,15 @@ object YarnClient extends SparkClientTrait { } private def setYarnQueue(submitRequest: SubmitRequest): Unit = { - if (submitRequest.hasExtra(KEY_SPARK_YARN_QUEUE_NAME)) { - submitRequest.appProperties.put(KEY_SPARK_YARN_QUEUE, submitRequest.getExtra(KEY_SPARK_YARN_QUEUE_NAME).asInstanceOf[String]) + if (submitRequest.hasExtra(SparkOptions.YARN_QUEUE_NAME.key)) { + submitRequest.appProperties.put( + SparkOptions.YARN_QUEUE.key, + submitRequest.getExtra(SparkOptions.YARN_QUEUE_NAME.key).asInstanceOf[String]) } - if (submitRequest.hasExtra(KEY_SPARK_YARN_QUEUE_LABEL)) { - submitRequest.appProperties.put(KEY_SPARK_YARN_AM_NODE_LABEL, submitRequest.getExtra(KEY_SPARK_YARN_QUEUE_LABEL).asInstanceOf[String]) - submitRequest.appProperties.put(KEY_SPARK_YARN_EXECUTOR_NODE_LABEL, submitRequest.getExtra(KEY_SPARK_YARN_QUEUE_LABEL).asInstanceOf[String]) + if (submitRequest.hasExtra(SparkOptions.YARN_QUEUE_LABEL.key)) { + val nodeLabel = submitRequest.getExtra(SparkOptions.YARN_QUEUE_LABEL.key).asInstanceOf[String] + submitRequest.appProperties.put(SparkOptions.YARN_AM_NODE_LABEL.key, nodeLabel) + submitRequest.appProperties.put(SparkOptions.YARN_EXECUTOR_NODE_LABEL.key, nodeLabel) } } } diff --git a/streampark-spark/streampark-spark-core/src/main/java/org/apache/streampark/spark/configuration/SparkRuntimeOptions.java b/streampark-spark/streampark-spark-core/src/main/java/org/apache/streampark/spark/configuration/SparkRuntimeOptions.java new file mode 100644 index 0000000000..32f8b11dc3 --- /dev/null +++ b/streampark-spark/streampark-spark-core/src/main/java/org/apache/streampark/spark/configuration/SparkRuntimeOptions.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.spark.configuration; + +import org.apache.streampark.common.configuration.ConfigOption; +import org.apache.streampark.common.configuration.ConfigOptions; + +/** Typed options consumed directly by StreamPark's Spark application runtime. */ +public final class SparkRuntimeOptions { + + public static final ConfigOption MAIN_CLASS = string("spark.main.class"); + public static final ConfigOption APPLICATION_NAME = string("spark.app.name"); + public static final ConfigOption BATCH_DURATION_SECONDS = + ConfigOptions.key("spark.batch.duration") + .intType() + .noDefaultValue() + .check(value -> value > 0, "batch duration must be greater than zero") + .withDescription("Spark Streaming batch duration in seconds.") + .build(); + + private SparkRuntimeOptions() { + } + + private static ConfigOption string(String key) { + return ConfigOptions.key(key) + .stringType() + .noDefaultValue() + .withDescription("Spark runtime option '" + key + "'.") + .build(); + } +} diff --git a/streampark-spark/streampark-spark-core/src/main/java/org/apache/streampark/spark/configuration/package-info.java b/streampark-spark/streampark-spark-core/src/main/java/org/apache/streampark/spark/configuration/package-info.java new file mode 100644 index 0000000000..af4d8fb06c --- /dev/null +++ b/streampark-spark/streampark-spark-core/src/main/java/org/apache/streampark/spark/configuration/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Typed configuration definitions owned by the Spark application runtime. */ +package org.apache.streampark.spark.configuration; diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/ParameterTool.java b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/ParameterTool.java deleted file mode 100644 index b6140f84b6..0000000000 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/ParameterTool.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.core.util; - -import org.apache.commons.lang3.math.NumberUtils; -import java.util.*; - - -public class ParameterTool { - - protected static final String NO_VALUE_KEY = "__NO_VALUE_KEY"; - - public static String getKeyFromArgs(String[] args, int index) { - String key; - if (args[index].startsWith("--")) { - key = args[index].substring(2); - } else if (args[index].startsWith("-")) { - key = args[index].substring(1); - } else { - throw new IllegalArgumentException( - String.format( - "Error parsing arguments '%s' on '%s'. Please prefix keys with -- or -.", - Arrays.toString(args), args[index])); - } - - if (key.isEmpty()) { - throw new IllegalArgumentException( - "The input " + Arrays.toString(args) + " contains an empty argument"); - } - - return key; - } - - public static Map fromArgs(String[] args) { - final Map map = new HashMap<>(args.length / 2); - - int i = 0; - while (i < args.length) { - final String key = getKeyFromArgs(args, i); - - if (key.isEmpty()) { - throw new IllegalArgumentException( - "The input " + Arrays.toString(args) + " contains an empty argument"); - } - - i += 1; // try to find the value - - if (i >= args.length) { - map.put(key, NO_VALUE_KEY); - } else if (NumberUtils.isCreatable(args[i])) { - map.put(key, args[i]); - i += 1; - } else if (args[i].startsWith("--") || args[i].startsWith("-")) { - // the argument cannot be a negative number because we checked earlier - // -> the next argument is a parameter name - map.put(key, NO_VALUE_KEY); - } else { - map.put(key, args[i]); - i += 1; - } - } - - return map; - } -} diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SQLContextUtil.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SQLContextUtil.scala deleted file mode 100644 index f486b5868e..0000000000 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SQLContextUtil.scala +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.spark.core.util - -import org.apache.spark.SparkContext -import org.apache.spark.sql.{SparkSession, SQLContext} - -/** SQLContext singleton */ -object SQLContextUtil { - - @transient private var instance: SQLContext = _ - @transient private var hiveContext: SQLContext = _ - - def getSqlContext(@transient sparkContext: SparkContext): SQLContext = { - if (instance == null) { - instance = SparkSession - .builder() - .config(sparkContext.getConf) - .getOrCreate() - .sqlContext - } - instance - } - - /** Get HiveContext */ - def getHiveContext(@transient sparkContext: SparkContext): SQLContext = { - if (hiveContext == null) { - hiveContext = SparkSession - .builder() - .config(sparkContext.getConf) - .enableHiveSupport() - .getOrCreate() - .sqlContext - } - hiveContext - } -} diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SparkSqlExecutor.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SparkSqlExecutor.scala index 595e8522ae..44f421e310 100644 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SparkSqlExecutor.scala +++ b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SparkSqlExecutor.scala @@ -17,9 +17,11 @@ package org.apache.streampark.spark.core.util -import org.apache.streampark.common.conf.ConfigKeys._ -import org.apache.streampark.common.util.{DeflaterUtils, Logger, PropertiesUtils} +import org.apache.streampark.common.configuration.{CommandLineParser, ConfigurationParser} +import org.apache.streampark.common.configuration.option.ApplicationOptions +import org.apache.streampark.common.util.{DeflaterUtils, Logger} import org.apache.streampark.common.util.Implicits._ +import org.apache.streampark.spark.configuration.SparkRuntimeOptions import org.apache.commons.lang3.StringUtils import org.apache.spark.SparkConf @@ -52,10 +54,14 @@ object SparkSqlExecutor extends Logger { val context = checkpoint match { case "" => - new StreamingContext(sparkSession.sparkContext, Seconds(sparkConf.get(KEY_SPARK_BATCH_DURATION).toInt)) + new StreamingContext( + sparkSession.sparkContext, + Seconds(sparkConf.get(SparkRuntimeOptions.BATCH_DURATION_SECONDS.key).toInt)) case checkpointPath => def createContext(): StreamingContext = - new StreamingContext(sparkSession.sparkContext, Seconds(sparkConf.get(KEY_SPARK_BATCH_DURATION).toInt)) + new StreamingContext( + sparkSession.sparkContext, + Seconds(sparkConf.get(SparkRuntimeOptions.BATCH_DURATION_SECONDS.key).toInt)) val tmpContext = StreamingContext.getOrCreate(checkpointPath, createContext _, createOnError = createOnError) tmpContext.checkpoint(checkpointPath) @@ -81,9 +87,8 @@ object SparkSqlExecutor extends Logger { } private def executeSql(args: Array[String], sparkSession: SparkSession): Unit = { - val parameterTool = ParameterTool.fromArgs(args) val sparkSql = { - val sql = parameterTool.get(KEY_SPARK_SQL()) + val sql = CommandLineParser.parse(args).get(ApplicationOptions.SQL) require(StringUtils.isNotBlank(sql), "Usage: spark sql cannot be null") Try(DeflaterUtils.unzipString(sql)) match { case Success(value) => value @@ -122,7 +127,7 @@ object SparkSqlExecutor extends Logger { case "--createOnError" :: _ :: tail => argv = tail case Nil => - case other :: value :: tail if other.startsWith(PARAM_PREFIX) => + case other :: value :: tail if other.startsWith(CommandLineParser.LONG_OPTION_PREFIX) => userArgs += other.drop(2) -> value argv = tail case tail => @@ -132,25 +137,18 @@ object SparkSqlExecutor extends Logger { } if (conf != null) { - val localConf = conf.split("\\.").last match { - case "conf" => PropertiesUtils.fromHoconFile(conf) - case "properties" => PropertiesUtils.fromPropertiesFile(conf) - case "yaml" | "yml" => PropertiesUtils.fromYamlFile(conf) - case _ => - throw new IllegalArgumentException( - "[StreamPark] Usage: config file error,must be [properties|yaml|conf]") - } + val localConf = ConfigurationParser.parse(java.nio.file.Paths.get(conf)).toMap.asScala localConf.foreach(arg => sparkConf.set(arg._1, arg._2)) } userArgs.foreach(arg => sparkConf.set(arg._1, arg._2)) - val appMain = sparkConf.get(KEY_SPARK_MAIN_CLASS, "org.apache.streampark.spark.cli.SqlClient") + val appMain = sparkConf.get(SparkRuntimeOptions.MAIN_CLASS.key, "org.apache.streampark.spark.cli.SqlClient") if (appMain == null) { - logError(s"[StreamPark] parameter: $KEY_SPARK_MAIN_CLASS must not be empty!") + logError(s"[StreamPark] parameter: ${SparkRuntimeOptions.MAIN_CLASS.key} must not be empty!") System.exit(1) } - val appName = sparkConf.get(KEY_SPARK_APP_NAME, null) match { + val appName = sparkConf.get(SparkRuntimeOptions.APPLICATION_NAME.key, null) match { case null | "" => appMain case name => name } diff --git a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SqlCommandParser.scala b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SqlCommandParser.scala index 9a9604faae..50a852d949 100644 --- a/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SqlCommandParser.scala +++ b/streampark-spark/streampark-spark-core/src/main/scala/org/apache/streampark/spark/core/util/SqlCommandParser.scala @@ -17,7 +17,7 @@ package org.apache.streampark.spark.core.util -import org.apache.streampark.common.conf.ConfigKeys.PARAM_PREFIX +import org.apache.streampark.common.configuration.CommandLineParser import org.apache.streampark.common.enums.SparkSqlValidationFailedType import org.apache.streampark.common.util.Logger @@ -395,7 +395,7 @@ case class SqlSegment(start: Int, end: Int, sql: String) object SqlSplitter { - private lazy val singleLineCommentPrefixList = Set[String](PARAM_PREFIX) + private lazy val singleLineCommentPrefixList = Set[String](CommandLineParser.LONG_OPTION_PREFIX) /** * Split whole text into multiple sql statements. Two Steps: Step 1, split the whole text into @@ -432,7 +432,7 @@ object SqlSplitter { lineNumber += 1 val line = scanner.nextLine().trim val nonEmpty = - StringUtils.isNotBlank(line) && !line.startsWith(PARAM_PREFIX) + StringUtils.isNotBlank(line) && !line.startsWith(CommandLineParser.LONG_OPTION_PREFIX) if (line.startsWith("/*")) { startComment = true hasComment = true @@ -590,7 +590,7 @@ object SqlSplitter { } private[this] def isSingleLineComment(text: String) = - text.trim.startsWith(PARAM_PREFIX) + text.trim.startsWith(CommandLineParser.LONG_OPTION_PREFIX) private[this] def isMultipleLineComment(text: String) = text.trim.startsWith("/*") && text.trim.endsWith("*/") From 7f6835d37e3ff645ae75f82bfb72bacfe15c5939 Mon Sep 17 00:00:00 2001 From: benjobs Date: Thu, 3 Sep 2026 23:31:49 +0800 Subject: [PATCH 2/9] [Build] Address security and reliability findings --- .../configuration/ConfigurationParser.java | 4 +- .../streampark/common/fs/FsOperator.java | 21 ++++ .../streampark/common/fs/HdfsOperator.java | 19 +++ .../streampark/common/fs/LfsOperator.java | 24 ++++ .../streampark/common/util/FileUtils.java | 6 +- .../common/util/FlinkConfigurationUtils.java | 3 +- .../streampark/common/util/HdfsUtils.java | 14 +++ .../streampark/common/util/PathUtils.java | 90 +++++--------- .../apache/streampark/common/util/Utils.java | 27 +++-- .../streampark/common/util/YamlParser.java | 2 +- .../streampark/common/fs/LfsOperatorTest.java | 19 ++- .../util/FlinkConfigurationUtilsTest.java | 6 +- .../streampark/common/util/PathUtilsTest.java | 79 ++++++++++++ .../streampark/common/util/UtilsTest.java | 2 +- .../console/base/util/WebUtils.java | 12 +- .../console/core/entity/FlinkSql.java | 7 +- .../impl/FlinkApplicationInfoServiceImpl.java | 19 ++- .../impl/SparkApplicationInfoServiceImpl.java | 19 ++- .../impl/FlinkSavepointServiceImpl.java | 2 +- .../core/service/impl/ProjectServiceImpl.java | 21 +++- .../core/service/impl/ProxyServiceImpl.java | 109 ++++++++++++----- .../service/impl/ResourceServiceImpl.java | 114 ++++++++++++------ .../console/base/util/WebUtilsTest.java | 4 +- .../console/core/entity/FlinkSqlTest.java | 12 ++ .../core/service/ResourceServiceTest.java | 48 ++++++-- .../service/impl/ProxyServiceImplTest.java | 80 ++++++++++++ .../packer/pipeline/YarnJarUploader.java | 2 +- 27 files changed, 591 insertions(+), 174 deletions(-) create mode 100644 streampark-common/src/test/java/org/apache/streampark/common/util/PathUtilsTest.java create mode 100644 streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/impl/ProxyServiceImplTest.java diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationParser.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationParser.java index f870995889..53426a0d01 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationParser.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigurationParser.java @@ -17,7 +17,6 @@ package org.apache.streampark.common.configuration; -import org.apache.streampark.common.util.PathUtils; import org.apache.streampark.common.util.YamlParser; import com.typesafe.config.Config; @@ -29,6 +28,7 @@ import java.io.Reader; import java.io.StringReader; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.util.LinkedHashMap; import java.util.Map; @@ -69,7 +69,7 @@ public static Configuration parse(Path path) { */ public static Configuration parse(Path path, ConfigurationFormat format) { Objects.requireNonNull(path, "path must not be null"); - try (InputStream inputStream = PathUtils.openFile(path.toString())) { + try (InputStream inputStream = Files.newInputStream(path)) { return parse(inputStream, format, path.toAbsolutePath().normalize().toString()); } catch (IOException e) { throw new ConfigException("Cannot read configuration file " + path, e); diff --git a/streampark-common/src/main/java/org/apache/streampark/common/fs/FsOperator.java b/streampark-common/src/main/java/org/apache/streampark/common/fs/FsOperator.java index 295151bf18..e5f7978d73 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/fs/FsOperator.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/fs/FsOperator.java @@ -69,6 +69,11 @@ public void move(String srcPath, String dstPath) { public String fileMd5(String path) { return LfsOperator.fileMd5(path); } + + @Override + public String fileSha256(String path) { + return LfsOperator.fileSha256(path); + } }; /** Scala-friendly HDFS operator singleton. */ @@ -118,6 +123,11 @@ public void move(String srcPath, String dstPath) { public String fileMd5(String path) { return HdfsOperator.fileMd5(path); } + + @Override + public String fileSha256(String path) { + return HdfsOperator.fileSha256(path); + } }; protected FsOperator() { @@ -177,4 +187,15 @@ public void copyDir(String srcPath, String dstPath) { public abstract void move(String srcPath, String dstPath); public abstract String fileMd5(String path); + + /** + * Returns the SHA-256 digest of a file for collision-resistant content comparison. + * + * @param path file path understood by this operator + * @return lowercase hexadecimal SHA-256 digest + * @throws UnsupportedOperationException when a custom operator does not provide digest support + */ + public String fileSha256(String path) { + throw new UnsupportedOperationException("SHA-256 is not supported by this file system"); + } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/fs/HdfsOperator.java b/streampark-common/src/main/java/org/apache/streampark/common/fs/HdfsOperator.java index a3d8918485..7489e2d285 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/fs/HdfsOperator.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/fs/HdfsOperator.java @@ -127,6 +127,25 @@ public static String fileMd5(String path) { } } + /** + * Returns the SHA-256 digest of an HDFS file. + * + * @param path HDFS file path + * @return lowercase hexadecimal SHA-256 digest + */ + public static String fileSha256(String path) { + if (path == null || path.isEmpty()) { + throw new IllegalArgumentException( + "[StreamPark] HdfsOperator.fileSha256: file must not be null."); + } + try { + return HdfsUtils.fileSha256(toHdfsPath(path)); + } catch (IOException e) { + throw new IllegalStateException( + "[StreamPark] Failed to compute SHA-256 for: " + path, e); + } + } + private static String toHdfsPath(String path) { if (path.startsWith("hdfs://")) { return path; diff --git a/streampark-common/src/main/java/org/apache/streampark/common/fs/LfsOperator.java b/streampark-common/src/main/java/org/apache/streampark/common/fs/LfsOperator.java index b5924532d6..bba4318308 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/fs/LfsOperator.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/fs/LfsOperator.java @@ -199,6 +199,30 @@ public static String fileMd5(String path) { } } + /** + * Returns the SHA-256 digest of a local file. + * + * @param path local file path + * @return lowercase hexadecimal SHA-256 digest + */ + public static String fileSha256(String path) { + if (path == null || path.isEmpty()) { + throw new IllegalArgumentException( + "[StreamPark] LFsOperator.fileSha256: file must not be null."); + } + File file = new File(path); + if (!file.exists()) { + throw new IllegalArgumentException( + "[StreamPark] LFsOperator.fileSha256: file must exist."); + } + try (FileInputStream inputStream = new FileInputStream(file)) { + return DigestUtils.sha256Hex(inputStream); + } catch (IOException e) { + throw new IllegalStateException( + "[StreamPark] Failed to compute SHA-256 for: " + path, e); + } + } + /** Force delete directory and recreate it. */ public static void mkCleanDirs(String path) { delete(path); diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java index 62266bb987..0e961fa1ae 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/FileUtils.java @@ -255,12 +255,8 @@ public static void readInputStream(InputStream in, byte[] array) throws IOExcept } } - public static String readFile(String filename) throws IOException { - return PathUtils.readFileAsString(filename); - } - public static String readFile(File file) throws IOException { - return readFile(file.getPath()); + return Files.readString(file.toPath(), StandardCharsets.UTF_8); } public static void writeFile(String content, File file) throws IOException { diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java index fca6768359..282a2fa043 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java @@ -284,7 +284,8 @@ private static Map loadLegacy(BufferedReader reader, String reso int lineNumber = 0; while ((line = reader.readLine()) != null) { lineNumber++; - String content = line.split("#", 2)[0].trim(); + int commentStart = line.indexOf('#'); + String content = (commentStart < 0 ? line : line.substring(0, commentStart)).trim(); if (content.isEmpty()) { continue; } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/HdfsUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/HdfsUtils.java index 4a0919933a..fbf581bd07 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/HdfsUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/HdfsUtils.java @@ -181,6 +181,20 @@ public static String fileMd5(String fileName) throws IOException { } } + /** + * Returns the SHA-256 digest of an HDFS file. + * + * @param fileName HDFS file path + * @return lowercase hexadecimal SHA-256 digest + * @throws IOException when the file cannot be read + */ + public static String fileSha256(String fileName) throws IOException { + Path path = getPath(fileName); + try (FSDataInputStream input = HadoopUtils.hdfs().open(path)) { + return DigestUtils.sha256Hex(input); + } + } + public static void downToLocal(String hdfsPath, String localPath) throws IOException { Path path = getPath(hdfsPath); FSDataInputStream input = HadoopUtils.hdfs().open(path); diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/PathUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/PathUtils.java index e9da9466fc..acdb6f2493 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/PathUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/PathUtils.java @@ -18,78 +18,44 @@ package org.apache.streampark.common.util; import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; +import java.util.Objects; -/** Path resolution helpers for user-supplied file locations. */ +/** Resolves untrusted paths within an explicitly trusted directory. */ public final class PathUtils { private PathUtils() { } - public static Path resolvePath(String filename) { - validateFilename(filename); - Path base = Paths.get("").toAbsolutePath().normalize(); - Path resolved = base.resolve(filename).normalize(); - if (!Paths.get(filename).isAbsolute() && !resolved.startsWith(base)) { - throw new IllegalArgumentException("invalid file path: " + filename); + /** + * Resolves an existing path without allowing it to escape {@code directory}. Absolute paths + * are accepted only when they remain inside the trusted directory. Both paths are resolved + * through symbolic links before the final containment check. + * + * @param directory trusted parent directory + * @param child untrusted relative or absolute path + * @return normalized path contained by the trusted directory + * @throws IOException when canonical path resolution fails or the child escapes the directory + */ + public static Path resolveChild(Path directory, String child) throws IOException { + Objects.requireNonNull(directory, "directory must not be null"); + if (child == null || child.trim().isEmpty()) { + throw new IOException("child path must not be blank"); } - return resolved; - } - public static InputStream openFile(String filename) throws IOException { - validateFilename(filename); - Path base = Paths.get("").toAbsolutePath().normalize(); - Path resolved = base.resolve(filename).normalize(); - if (!Paths.get(filename).isAbsolute() && !resolved.startsWith(base)) { - throw new IOException("invalid file path: " + filename); + Path lexicalRoot = directory.toAbsolutePath().normalize(); + Path requested = Path.of(child); + Path candidate = requested.isAbsolute() + ? requested.normalize() + : lexicalRoot.resolve(requested).normalize(); + if (!candidate.startsWith(lexicalRoot)) { + throw new IOException("Path escapes trusted directory: " + child); } - return Files.newInputStream(resolved); // NOSONAR javasecurity:S2083 - path validated via Path.resolve above - } - - public static String readFileAsString(String filename) throws IOException { - Path resolved = resolvePath(filename); - // NOSONAR javasecurity:S2083 - path validated via - // Path.resolve above - return Files.readString(resolved, StandardCharsets.UTF_8); - } - - public static Path resolveJarPath(java.net.URL jar) throws IOException { - try { - String location = jar.toString(); - if (location.contains("..")) { - throw new IOException("JAR file path is invalid " + jar); - } - Path base = Paths.get("").toAbsolutePath().normalize(); - return base.resolve(Paths.get(jar.toURI())).normalize(); - } catch (Exception e) { - throw new IOException("JAR file path is invalid " + jar, e); - } - } - - public static InputStream openJarFile(java.net.URL jar) throws IOException { - String location = jar.toString(); - if (location.contains("..")) { - throw new IOException("JAR file path is invalid " + jar); - } - try { - Path base = Paths.get("").toAbsolutePath().normalize(); - Path resolved = base.resolve(Paths.get(jar.toURI())).normalize(); - return Files.newInputStream(resolved); // NOSONAR javasecurity:S2083 - path validated via Path.resolve above - } catch (Exception e) { - throw new IOException("JAR file path is invalid " + jar, e); - } - } - - private static void validateFilename(String filename) { - if (filename == null || filename.isEmpty()) { - throw new IllegalArgumentException("filename must not be blank"); - } - if (filename.contains("..")) { - throw new IllegalArgumentException("invalid file path: " + filename); + Path root = lexicalRoot.toRealPath(); + Path resolved = candidate.toRealPath(); + if (!resolved.startsWith(root)) { + throw new IOException("Path escapes trusted directory: " + child); } + return resolved; } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java index 902da5660b..b5491031cd 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/Utils.java @@ -21,12 +21,11 @@ import org.apache.commons.lang3.StringUtils; -import java.io.BufferedInputStream; import java.io.File; import java.io.Flushable; import java.io.IOException; -import java.io.InputStream; import java.net.HttpURLConnection; +import java.net.URISyntaxException; import java.net.URL; import java.time.Duration; import java.time.LocalDateTime; @@ -36,7 +35,7 @@ import java.util.Properties; import java.util.UUID; import java.util.concurrent.locks.LockSupport; -import java.util.jar.JarInputStream; +import java.util.jar.JarFile; import java.util.jar.Manifest; import java.util.regex.Pattern; @@ -84,9 +83,16 @@ public static String uuid() { } public static void requireCheckJarFile(URL jar) throws IOException { - try ( - InputStream in = PathUtils.openJarFile(jar); - JarInputStream ignored = new JarInputStream(new BufferedInputStream(in))) { + if (!"file".equalsIgnoreCase(jar.getProtocol())) { + throw new IOException("Only local JAR files are supported: " + jar); + } + File jarFile; + try { + jarFile = new File(jar.toURI()); + } catch (URISyntaxException e) { + throw new IOException("Invalid JAR file URL: " + jar, e); + } + try (JarFile ignored = new JarFile(jarFile)) { // verify jar is readable } catch (IOException e) { throw new IOException("Error while opening jar file '" + jar + "'", e); @@ -94,16 +100,17 @@ public static void requireCheckJarFile(URL jar) throws IOException { } public static Manifest getJarManifest(File jarFile) throws IOException { - try ( - InputStream in = PathUtils.openJarFile(jarFile.toURI().toURL()); - JarInputStream jarInputStream = new JarInputStream(new BufferedInputStream(in))) { - return jarInputStream.getManifest(); + try (JarFile jar = new JarFile(jarFile)) { + return jar.getManifest(); } } public static String getJarManClass(File jarFile) { try { Manifest manifest = getJarManifest(jarFile); + if (manifest == null) { + return null; + } String mainClass = manifest.getMainAttributes().getValue("Main-Class"); if (mainClass == null) { mainClass = manifest.getMainAttributes().getValue("program-class"); diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java b/streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java index cad042f6bc..23cd3eff40 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java @@ -65,7 +65,7 @@ public final class YamlParser { private static final String LENIENT_SCALAR_PREFIXES = "!&*%@`|>"; private static final Pattern EMPTY_VALUE_PROPERTY = - Pattern.compile("^(\\s*)([^\\s#][^:]*):(?:\\s*(#.*)?)?$"); + Pattern.compile("^(\\s*)([^\\s#][^:]*):\\s*(#.*)?$"); private static final Pattern STRUCTURED_CHILD = Pattern.compile("^[^#\\-][^:]*:(?:$|\\s.*)$"); private static final Pattern BLOCK_SCALAR_HEADER = diff --git a/streampark-common/src/test/java/org/apache/streampark/common/fs/LfsOperatorTest.java b/streampark-common/src/test/java/org/apache/streampark/common/fs/LfsOperatorTest.java index 789f04ab29..029bd06fea 100644 --- a/streampark-common/src/test/java/org/apache/streampark/common/fs/LfsOperatorTest.java +++ b/streampark-common/src/test/java/org/apache/streampark/common/fs/LfsOperatorTest.java @@ -25,7 +25,9 @@ import java.io.File; import java.io.FileInputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.List; @@ -393,9 +395,24 @@ void testMove() throws Exception { } @Test - void testfileMd5() { + void rejectInvalidDigestPaths() { assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileMd5(null)); assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileMd5("")); assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileMd5("ttt/144514.dat")); + assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileSha256(null)); + assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileSha256("")); + assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileSha256("ttt/144514.dat")); + } + + @Test + void computeSha256() throws Exception { + withTempDir( + outputDir -> { + byte[] content = "streampark".getBytes(StandardCharsets.UTF_8); + Path file = Paths.get(outputDir, "artifact.jar"); + Files.write(file, content); + + assertEquals(DigestUtils.sha256Hex(content), LfsOperator.fileSha256(file.toString())); + }); } } diff --git a/streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java b/streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java index 050670566b..21a4a70db9 100644 --- a/streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java +++ b/streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java @@ -160,7 +160,11 @@ void requireConfigYamlFromFlink2() throws Exception { void readLegacyFileAsUtf8() throws Exception { Path file = temporaryDirectory.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME); - Files.writeString(file, "pipeline.name: \u4e2d\u6587\u4f5c\u4e1a\n", StandardCharsets.UTF_8); + Files.writeString( + file, + "# comment-only lines must be ignored\n" + + "pipeline.name: \u4e2d\u6587\u4f5c\u4e1a\n", + StandardCharsets.UTF_8); Map configuration = FlinkConfigurationUtils.loadConfigurationFromFile(file.toFile()); diff --git a/streampark-common/src/test/java/org/apache/streampark/common/util/PathUtilsTest.java b/streampark-common/src/test/java/org/apache/streampark/common/util/PathUtilsTest.java new file mode 100644 index 0000000000..41a8000154 --- /dev/null +++ b/streampark-common/src/test/java/org/apache/streampark/common/util/PathUtilsTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.common.util; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies containment guarantees for request-supplied child paths. */ +class PathUtilsTest { + + @TempDir + Path directory; + + @TempDir + Path outsideDirectory; + + @Test + void resolveContainedChild() throws Exception { + Path child = Files.createDirectory(directory.resolve("module")); + + assertThat(PathUtils.resolveChild(directory, "module")) + .isEqualTo(child.toRealPath()); + } + + @Test + void resolveAbsoluteChild() throws Exception { + Path child = Files.createFile(directory.resolve("application.yaml")); + + assertThat(PathUtils.resolveChild(directory, child.toString())) + .isEqualTo(child.toRealPath()); + } + + @Test + void rejectEscapingChild() { + assertThatThrownBy(() -> PathUtils.resolveChild(directory, "../outside")) + .isInstanceOf(java.io.IOException.class) + .hasMessageContaining("escapes trusted directory"); + } + + @Test + void rejectAbsoluteOutsideRoot() throws Exception { + Path outside = Files.createFile(outsideDirectory.resolve("outside.yaml")); + + assertThatThrownBy(() -> PathUtils.resolveChild(directory, outside.toString())) + .isInstanceOf(java.io.IOException.class) + .hasMessageContaining("escapes trusted directory"); + } + + @Test + void rejectSymlinkOutsideRoot() throws Exception { + Path outside = Files.createFile(outsideDirectory.resolve("linked-target.yaml")); + Path link = Files.createSymbolicLink(directory.resolve("linked.yaml"), outside); + + assertThatThrownBy(() -> PathUtils.resolveChild(directory, link.toString())) + .isInstanceOf(java.io.IOException.class) + .hasMessageContaining("escapes trusted directory"); + } +} diff --git a/streampark-common/src/test/java/org/apache/streampark/common/util/UtilsTest.java b/streampark-common/src/test/java/org/apache/streampark/common/util/UtilsTest.java index 4ef6b51e34..e545d2e674 100644 --- a/streampark-common/src/test/java/org/apache/streampark/common/util/UtilsTest.java +++ b/streampark-common/src/test/java/org/apache/streampark/common/util/UtilsTest.java @@ -63,7 +63,7 @@ void requiredShouldThrowIllegalArgumentExceptionIfConditionIsFalse() { } @Test - void requireCheckJarFileShouldThrowIOExceptionIfJarFilePathIsInvalid() throws Exception { + void rejectNonFileJarUrl() throws Exception { URL jar = new URL("http://host/file"); IOException ex = assertThrows(IOException.class, () -> Utils.requireCheckJarFile(jar)); assertTrue(ex.getMessage().contains("http://host/file")); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java index cda738879e..371a0682a7 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java @@ -24,6 +24,7 @@ import com.baomidou.mybatisplus.core.toolkit.StringPool; import okhttp3.Headers; +import okhttp3.HttpUrl; import okhttp3.MediaType; import okhttp3.Request; import okhttp3.RequestBody; @@ -134,9 +135,15 @@ public static File getPluginDir() { return getAppDir(PLUGINS); } - /** Proxies a request and streams the upstream response to the servlet response. */ + /** + * Proxies a request to a prevalidated upstream URL and streams its response. + * + *

The caller constructs {@link HttpUrl} from a trusted upstream authority. Request paths and + * query parameters must be added through {@link HttpUrl.Builder}, which prevents them from + * being reinterpreted as a different host. + */ public static void http( - String url, + HttpUrl url, HttpServletRequest request, HttpServletResponse response) throws IOException { Headers.Builder headersBuilder = new Headers.Builder(); @@ -191,7 +198,6 @@ private static void writeProxyResponse( Response upstream, HttpServletResponse downstream) throws IOException { copyProxyResponseHeaders(upstream, downstream); downstream.setStatus(upstream.code()); - downstream.setHeader("Access-Control-Allow-Origin", "*"); if (HttpMethod.HEAD.matches(upstream.request().method())) { return; } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkSql.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkSql.java index e986ac33b1..811238b9e9 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkSql.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/FlinkSql.java @@ -106,7 +106,9 @@ public void decode() { /** Copies decoded SQL to a Flink application using the API transport representation. */ public void applyToApplication(FlinkApplication application) { application.setFlinkSql( - Base64.getEncoder().encodeToString(this.sql.getBytes(StandardCharsets.UTF_8))); + this.sql == null + ? null + : Base64.getEncoder().encodeToString(this.sql.getBytes(StandardCharsets.UTF_8))); application.setDependency(this.dependency); application.setTeamResource(this.teamResource); application.setSqlId(this.id); @@ -149,6 +151,9 @@ public ChangeTypeEnum checkChange(FlinkSql target) { /** Replaces SQL with the Base64-encoded plain text required by the API. */ public void base64Encode() { decode(); + if (this.sql == null) { + return; + } this.sql = Base64.getEncoder().encodeToString(this.sql.getBytes(StandardCharsets.UTF_8)); } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java index 3ecadd9a15..5dff831367 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationInfoServiceImpl.java @@ -17,10 +17,12 @@ package org.apache.streampark.console.core.service.application.impl; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.util.ExceptionUtils; import org.apache.streampark.common.util.HadoopUtils; +import org.apache.streampark.common.util.PathUtils; import org.apache.streampark.common.util.Utils; import org.apache.streampark.common.util.YarnUtils; import org.apache.streampark.console.base.exception.ApiAlertException; @@ -65,6 +67,8 @@ import java.io.IOException; import java.io.Serializable; import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.Base64; import java.util.EnumSet; import java.util.HashMap; @@ -413,7 +417,7 @@ public String k8sStartLog(Long id, Integer offset, Integer limit) throws Excepti public String getYarnName(String appConfig) { String[] args = new String[2]; args[0] = "--name"; - args[1] = appConfig; + args[1] = resolveProjectConfig(appConfig).toString(); return FlinkShellCommandBuilder.read(args); } @@ -458,9 +462,18 @@ public AppExistsStateEnum checkExists(FlinkApplication appParam) { @Override public String readConf(String appConfig) throws IOException { - File file = new File(appConfig); + File file = resolveProjectConfig(appConfig).toFile(); String conf = org.apache.streampark.common.util.FileUtils.readFile(file); - return Base64.getEncoder().encodeToString(conf.getBytes()); + return Base64.getEncoder().encodeToString(conf.getBytes(StandardCharsets.UTF_8)); + } + + /** Resolves a client-selected configuration within managed project distributions. */ + private Path resolveProjectConfig(String appConfig) { + try { + return PathUtils.resolveChild(Path.of(Workspace.APP_LOCAL_DIST), appConfig); + } catch (IOException exception) { + throw new ApiDetailException("Invalid project configuration path", exception); + } } @Override diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationInfoServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationInfoServiceImpl.java index 82d320ff08..e7c3515b0d 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationInfoServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationInfoServiceImpl.java @@ -17,10 +17,12 @@ package org.apache.streampark.console.core.service.application.impl; +import org.apache.streampark.common.configuration.Workspace; import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.SparkDeployMode; import org.apache.streampark.common.util.ExceptionUtils; import org.apache.streampark.common.util.HadoopUtils; +import org.apache.streampark.common.util.PathUtils; import org.apache.streampark.common.util.YarnUtils; import org.apache.streampark.console.base.exception.ApiDetailException; import org.apache.streampark.console.base.exception.ApplicationException; @@ -51,6 +53,8 @@ import java.io.File; import java.io.IOException; import java.io.Serializable; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.util.Base64; import java.util.EnumSet; import java.util.HashMap; @@ -247,7 +251,7 @@ public List getYarnAppReport(String appName) { public String getYarnName(String appConfig) { String[] args = new String[2]; args[0] = "--name"; - args[1] = appConfig; + args[1] = resolveProjectConfig(appConfig).toString(); return FlinkShellCommandBuilder.read(args); } @@ -305,9 +309,18 @@ private boolean existsByAppName(String jobName) { @Override public String readConf(String appConfig) throws IOException { - File file = new File(appConfig); + File file = resolveProjectConfig(appConfig).toFile(); String conf = org.apache.streampark.common.util.FileUtils.readFile(file); - return Base64.getEncoder().encodeToString(conf.getBytes()); + return Base64.getEncoder().encodeToString(conf.getBytes(StandardCharsets.UTF_8)); + } + + /** Resolves a client-selected configuration within managed project distributions. */ + private Path resolveProjectConfig(String appConfig) { + try { + return PathUtils.resolveChild(Path.of(Workspace.APP_LOCAL_DIST), appConfig); + } catch (IOException exception) { + throw new ApiDetailException("Invalid project configuration path", exception); + } } private Boolean checkJobName(String jobName) { diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java index 7fb55a8488..9fc8d14635 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java @@ -473,7 +473,7 @@ private void expire(FlinkSavepoint entity) { .eq(FlinkSavepoint::getType, CHECKPOINT.get()) .orderByDesc(FlinkSavepoint::getTriggerTime) .page( - new Page<>(1, cpThreshold + 1)); + new Page<>(1, cpThreshold + 1L)); if (CollectionUtils.isEmpty(savepointPage.getRecords()) || savepointPage.getRecords().size() <= cpThreshold) { diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java index f00dd0a726..1114607fab 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java @@ -23,6 +23,7 @@ import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.CompletableFutureUtils; import org.apache.streampark.common.util.FileUtils; +import org.apache.streampark.common.util.PathUtils; import org.apache.streampark.console.base.config.ConsoleOptions; import org.apache.streampark.console.base.domain.RestRequest; import org.apache.streampark.console.base.exception.ApiAlertException; @@ -59,6 +60,7 @@ import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -268,8 +270,11 @@ public List listModules(Long id) { public List listJars(Project project) { ApiAlertException.throwIfNull( project.getModule(), "Project module can't be null, please check."); - File projectModuleDir = new File(project.getDistHome(), project.getModule()); - return Arrays.stream(Objects.requireNonNull(projectModuleDir.listFiles())) + File[] files = resolveModule(project).listFiles(); + if (files == null) { + return Collections.emptyList(); + } + return Arrays.stream(files) .map(File::getName) .filter(name -> name.endsWith(Constants.JAR_SUFFIX)) .collect(Collectors.toList()); @@ -305,7 +310,7 @@ public boolean exists(Project project) { @Override public List> listConf(Project project) { try { - File file = new File(project.getDistHome(), project.getModule()); + File file = resolveModule(project); File unzipFile = new File(file.getAbsolutePath().replaceAll(".tar.gz", "")); if (!unzipFile.exists()) { GZipUtils.deCompress(file.getAbsolutePath(), file.getParentFile().getAbsolutePath()); @@ -323,6 +328,16 @@ public List> listConf(Project project) { return null; } + /** Resolves a request-supplied module below the project's distribution directory. */ + private File resolveModule(Project project) { + try { + Path distHome = project.getDistHome().toPath(); + return PathUtils.resolveChild(distHome, project.getModule()).toFile(); + } catch (IOException e) { + throw new ApiDetailException("Invalid project module path", e); + } + } + private void eachFile(File file, List> list, Boolean isRoot) { if (file != null && file.exists() && file.listFiles() != null) { if (isRoot) { diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java index ba68a95b77..ced07f2589 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java @@ -34,6 +34,7 @@ import org.apache.hadoop.security.UserGroupInformation; import lombok.extern.slf4j.Slf4j; +import okhttp3.HttpUrl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Service; @@ -43,7 +44,10 @@ import java.io.IOException; import java.io.PrintWriter; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; import java.security.PrivilegedExceptionAction; +import java.util.Arrays; /** Routes console proxy requests to Flink, YARN, and remote cluster endpoints. */ @Slf4j @@ -69,9 +73,11 @@ public void proxyFlink( case YARN_PER_JOB: case YARN_APPLICATION: case YARN_SESSION: - url = "/proxy/" + app.getClusterId(); - url += getRequestURL(request, "/proxy/flink/" + app.getId()); - proxyYarnRequest(request, response, url); + url = YarnUtils.getRMWebAppProxyURL() + "/proxy/" + app.getClusterId(); + proxyYarnRequest( + request, + response, + proxyUrl(url, request, "/proxy/flink/" + app.getId())); return; case REMOTE: FlinkCluster cluster = flinkClusterService.getById(app.getFlinkClusterId()); @@ -90,7 +96,10 @@ public void proxyFlink( unavailableResponse(response, "The flink job manager url is not ready"); return; } - proxyRequest(request, response, url + getRequestURL(request, "/proxy/flink/" + app.getId())); + proxyRequest( + request, + response, + proxyUrl(url, request, "/proxy/flink/" + app.getId())); } @Override @@ -101,9 +110,11 @@ public void proxySpark( switch (app.getDeployModeEnum()) { case YARN_CLIENT: case YARN_CLUSTER: - String url = "/proxy/" + app.getClusterId(); - url += getRequestURL(request, "/proxy/spark/" + app.getId()); - proxyYarnRequest(request, response, url); + String url = YarnUtils.getRMWebAppProxyURL() + "/proxy/" + app.getClusterId(); + proxyYarnRequest( + request, + response, + proxyUrl(url, request, "/proxy/spark/" + app.getId())); return; default: throw new UnsupportedOperationException( @@ -121,9 +132,11 @@ public void proxyYarn( unavailableResponse(response, "The yarn application id is null."); return; } - String url = "/proxy/" + yarnId + "/"; - url += getRequestURL(request, "/proxy/yarn/" + log.getId()); - proxyYarnRequest(request, response, url); + String url = YarnUtils.getRMWebAppProxyURL() + "/proxy/" + yarnId + "/"; + proxyYarnRequest( + request, + response, + proxyUrl(url, request, "/proxy/yarn/" + log.getId())); } @Override @@ -136,7 +149,10 @@ public void proxyHistory( unavailableResponse(response, "The jobManager url is null."); return; } - proxyRequest(request, response, url + getRequestURL(request, "/proxy/history/" + log.getId())); + proxyRequest( + request, + response, + proxyUrl(url, request, "/proxy/history/" + log.getId())); } @Override @@ -155,17 +171,17 @@ public void proxyFlinkCluster( return; } - url += getRequestURL(request, "/proxy/flink_cluster/" + clusterId); + HttpUrl target = proxyUrl(url, request, "/proxy/flink_cluster/" + clusterId); switch (cluster.getFlinkDeployModeEnum()) { case YARN_PER_JOB: case YARN_APPLICATION: case YARN_SESSION: - proxyYarnRequest(request, response, url); + proxyYarnRequest(request, response, target); return; case REMOTE: case KUBERNETES_NATIVE_APPLICATION: case KUBERNETES_NATIVE_SESSION: - proxyRequest(request, response, url); + proxyRequest(request, response, target); return; default: throw new UnsupportedOperationException( @@ -177,7 +193,7 @@ public void proxyFlinkCluster( private void proxyRequest( HttpServletRequest request, HttpServletResponse response, - String url) throws Exception { + HttpUrl url) throws Exception { try { WebUtils.http(url, request, response); } catch (Exception e) { @@ -192,10 +208,9 @@ private void proxyRequest( private void proxyYarnRequest( HttpServletRequest request, HttpServletResponse response, - String path) throws Exception { - String url = isAbsoluteUrl(path) ? path : YarnUtils.getRMWebAppProxyURL() + path; + HttpUrl url) throws Exception { if (YarnUtils.hasYarnHttpKerberosAuth()) { - final String kerberosUrl = url; + final HttpUrl kerberosUrl = url; UserGroupInformation ugi = HadoopUtils.getUgi(); try { ugi.doAs( @@ -203,6 +218,9 @@ private void proxyYarnRequest( proxyRequest(request, response, kerberosUrl); return null; }); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw e; } catch (Exception e) { log.error("Kerberos YARN proxy request failed for {}", request.getRequestURI(), e); if (!response.isCommitted()) { @@ -213,22 +231,57 @@ private void proxyYarnRequest( } if (YarnUtils.hasYarnHttpSimpleAuth()) { - String urlTemplate = - StringUtils.isNotBlank(request.getQueryString()) ? "%s&user.name=%s" : "%s?user.name=%s"; - url = String.format(urlTemplate, url, HadoopConfigUtils.hadoopUserName()); + url = url.newBuilder() + .addQueryParameter("user.name", HadoopConfigUtils.hadoopUserName()) + .build(); } proxyRequest(request, response, url); } - private String getRequestURL(HttpServletRequest request, String replaceString) { - String url = - request.getRequestURI() - + (request.getQueryString() != null ? "?" + request.getQueryString() : ""); - return url.replace(replaceString, ""); + /** Builds a target while keeping request data out of the upstream authority. */ + HttpUrl proxyUrl( + String baseUrl, + HttpServletRequest request, + String proxyPrefix) { + HttpUrl base = HttpUrl.parse(baseUrl); + if (base == null || !base.username().isEmpty() || !base.password().isEmpty()) { + throw new IllegalArgumentException("Invalid proxy upstream URL"); + } + + String route = request.getContextPath() + proxyPrefix; + String requestPath = request.getRequestURI(); + if (!requestPath.equals(route) && !requestPath.startsWith(route + "/")) { + throw new IllegalArgumentException("Request path is outside the proxy route"); + } + String path = requestPath.substring(route.length()); + rejectPathTraversal(path); + + HttpUrl.Builder target = base.newBuilder().fragment(null); + if (!path.isEmpty()) { + target.addEncodedPathSegments(path.substring(1)); + } + target.encodedQuery(request.getQueryString()); + return target.build(); } - private boolean isAbsoluteUrl(String url) { - return StringUtils.startsWithAny(url, "http://", "https://"); + /** Rejects traversal before OkHttp normalizes encoded path segments. */ + private void rejectPathTraversal(String path) { + String decoded = path; + for (int pass = 0; pass < 3; pass++) { + String normalized = decoded.replace('\\', '/'); + boolean hasTraversal = + Arrays.stream(normalized.split("/", -1)) + .anyMatch(segment -> ".".equals(segment) || "..".equals(segment)); + if (hasTraversal) { + throw new IllegalArgumentException("Proxy path traversal is not allowed"); + } + String next = URLDecoder.decode(decoded, StandardCharsets.UTF_8); + if (next.equals(decoded)) { + return; + } + decoded = next; + } + throw new IllegalArgumentException("Proxy path contains excessive encoding"); } private void unavailableResponse(HttpServletResponse response, String body) throws IOException { diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ResourceServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ResourceServiceImpl.java index 91487b2d7f..3843d54865 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ResourceServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ResourceServiceImpl.java @@ -22,6 +22,7 @@ import org.apache.streampark.common.fs.FsOperator; import org.apache.streampark.common.fs.LfsOperator; import org.apache.streampark.common.util.ExceptionUtils; +import org.apache.streampark.common.util.PathUtils; import org.apache.streampark.common.util.Utils; import org.apache.streampark.console.base.domain.RestRequest; import org.apache.streampark.console.base.exception.ApiAlertException; @@ -47,7 +48,6 @@ import org.apache.streampark.flink.packer.maven.MavenTool; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.table.factories.Factory; @@ -70,6 +70,10 @@ import java.io.InputStreamReader; import java.net.URL; import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; @@ -78,7 +82,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Scanner; import java.util.ServiceLoader; @@ -147,10 +150,9 @@ public void addResource(Resource resource) throws Exception { if (!jars.isEmpty()) { String resourcePath = jars.get(0); - resource.setResourcePath(resourcePath); - // copy jar to team upload directory - String upFile = resourcePath.split(":", 2)[1]; - transferTeamResource(resource.getTeamId(), upFile); + String[] resourceEntry = splitResourceEntry(resourcePath); + String storedPath = transferUploadedResource(resource.getTeamId(), resourceEntry[1]); + resource.setResourcePath(resourceEntry[0] + ":" + storedPath); } resource.setCreatorId(ServiceHelper.getUserId()); @@ -203,8 +205,10 @@ public void updateResource(Resource resource) { Dependency dependency = Dependency.toDependency(resource.getResource()); if (!dependency.getJar().isEmpty()) { - String jarFile = dependency.getJar().get(0).split(":", 2)[1]; - transferTeamResource(findResource.getTeamId(), jarFile); + String[] resourceEntry = splitResourceEntry(dependency.getJar().get(0)); + String storedPath = + transferUploadedResource(findResource.getTeamId(), resourceEntry[1]); + findResource.setResourcePath(resourceEntry[0] + ":" + storedPath); } } @@ -221,18 +225,19 @@ public void remove(Long id) { Resource findResource = getById(id); checkOrElseAlert(findResource); - String filePath = String.format( - "%s/%d/%s", - Workspace.LOCAL.uploads, - findResource.getTeamId(), - findResource.getResourceName()); - - if (!new File(filePath).exists() && StringUtils.isNotBlank(findResource.getFilePath())) { - filePath = findResource.getFilePath(); + if (StringUtils.isNotBlank(findResource.getFilePath())) { + String teamUploads = String.format("%s/%d", Workspace.LOCAL.uploads, findResource.getTeamId()); + try { + Path file = + PathUtils.resolveChild(Path.of(teamUploads), findResource.getFilePath()); + FsOperator.lfs().delete(file.toString()); + } catch (NoSuchFileException ignored) { + // Deletion is idempotent when the managed resource file is already absent. + } catch (IOException exception) { + throw new ApiDetailException("Invalid stored resource path", exception); + } } - FsOperator.lfs().delete(filePath); - this.removeById(id); } @@ -255,31 +260,44 @@ public void changeOwnership(Long userId, Long targetUserId) { } /** - * @param file - * @return + * Stores an uploaded resource under a server-generated temporary filename. + * + * @param file uploaded JAR or Python resource + * @return detected main class and server-side temporary path */ @Override public UploadResponse upload(MultipartFile file) throws IOException { - File temp = WebUtils.getAppTempDir(); - String fileName = FilenameUtils.getName(Objects.requireNonNull(file.getOriginalFilename())); - File saveFile = new File(temp, fileName); - if (!saveFile.exists()) { - // save file to temp dir - try { - file.transferTo(saveFile); - } catch (Exception e) { - throw new ApiDetailException(e); - } + String originalName = file.getOriginalFilename(); + ApiAlertException.throwIfTrue( + StringUtils.isBlank(originalName), "Uploaded resource name must not be blank"); + ApiAlertException.throwIfTrue(file.isEmpty(), "Uploaded resource must not be empty"); + boolean jar = StringUtils.endsWithIgnoreCase(originalName, ".jar"); + boolean python = StringUtils.endsWithIgnoreCase(originalName, ".py"); + ApiAlertException.throwIfTrue(!jar && !python, "Only JAR and Python resources are supported"); + String suffix = python ? ".py" : ".jar"; + Path temp = WebUtils.getAppTempDir().toPath(); + Files.createDirectories(temp); + Path savePath = Files.createTempFile(temp, "upload-", suffix); + File saveFile = savePath.toFile(); + try (InputStream input = file.getInputStream()) { + Files.copy(input, savePath, StandardCopyOption.REPLACE_EXISTING); + } catch (Exception e) { + Files.deleteIfExists(savePath); + throw new ApiDetailException(e); } String mainClass = null; - try { - mainClass = Utils.getJarManClass(saveFile); - } catch (Exception ignored) { + if (jar) { + try { + Utils.requireCheckJarFile(saveFile.toURI().toURL()); + mainClass = Utils.getJarManClass(saveFile); + } catch (Exception exception) { + Files.deleteIfExists(savePath); + throw new ApiDetailException("Uploaded JAR is invalid", exception); + } } - String path = saveFile.getAbsolutePath(); UploadResponse uploadResponse = new UploadResponse(); uploadResponse.setMainClass(mainClass); - uploadResponse.setPath(path); + uploadResponse.setPath(saveFile.getAbsolutePath()); return uploadResponse; } @@ -463,7 +481,30 @@ private File getResourceJar(Resource resource) throws Exception { } } - private void transferTeamResource(Long teamId, String resourcePath) { + /** Transfers a client-selected upload only after resolving it below the managed temp root. */ + private String transferUploadedResource(Long teamId, String resourcePath) { + try { + Path temp = WebUtils.getAppTempDir().toPath(); + Path upload = PathUtils.resolveChild(temp, resourcePath); + return transferTeamResource(teamId, upload.toString()); + } catch (IOException exception) { + throw new ApiDetailException("Invalid uploaded resource path", exception); + } + } + + /** Validates the display-name and path pair used by uploaded resource requests. */ + private String[] splitResourceEntry(String resourcePath) { + String[] entry = resourcePath.split(":", 2); + ApiAlertException.throwIfTrue( + entry.length != 2 + || StringUtils.isBlank(entry[0]) + || StringUtils.isBlank(entry[1]), + "Invalid uploaded resource entry"); + return entry; + } + + /** Copies a trusted local artifact into durable team storage and returns its final path. */ + private String transferTeamResource(Long teamId, String resourcePath) { String teamUploads = String.format("%s/%d", Workspace.LOCAL.uploads, teamId); if (!FsOperator.lfs().exists(teamUploads)) { FsOperator.lfs().mkdirs(teamUploads); @@ -474,6 +515,7 @@ private void transferTeamResource(Long teamId, String resourcePath) { localJar.exists(), "Missing file: " + resourcePath + ", please upload again"); FsOperator.lfs() .upload(localJar.getAbsolutePath(), teamUploadJar.getAbsolutePath(), false, true); + return teamUploadJar.getAbsolutePath(); } private void checkOrElseAlert(Resource resource) { diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/WebUtilsTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/WebUtilsTest.java index f66622a37b..653838b2de 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/WebUtilsTest.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/WebUtilsTest.java @@ -19,6 +19,7 @@ import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpServer; +import okhttp3.HttpUrl; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -83,7 +84,7 @@ void forwardProxyRequestAndResponse() throws Exception { request.addHeader("Connection", "keep-alive, X-Request-Hop"); MockHttpServletResponse response = new MockHttpServletResponse(); - WebUtils.http(endpoint, request, response); + WebUtils.http(HttpUrl.get(endpoint), request, response); assertThat(method.get()).isEqualTo("POST"); assertThat(requestBody.get()).isEqualTo("{\"name\":\"streampark\"}"); @@ -93,6 +94,7 @@ void forwardProxyRequestAndResponse() throws Exception { assertThat(response.getHeader("X-Upstream")).isEqualTo("ok"); assertThat(response.getHeader("Allow")).isEqualTo("GET, POST"); assertThat(response.getHeader("X-Upstream-Hop")).isNull(); + assertThat(response.getHeader("Access-Control-Allow-Origin")).isNull(); assertThat(response.getContentAsString()).isEqualTo("upstream response"); } diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/entity/FlinkSqlTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/entity/FlinkSqlTest.java index f3234ceb2c..b367986c9a 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/entity/FlinkSqlTest.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/entity/FlinkSqlTest.java @@ -61,6 +61,18 @@ void shouldEncodePersistedSqlForHistoryApi() { assertThat(decodeTransportValue(flinkSql.getSql())).isEqualTo(SQL); } + @Test + void keepNullSqlUnchanged() { + FlinkSql flinkSql = flinkSql(null); + FlinkApplication application = new FlinkApplication(); + + flinkSql.base64Encode(); + flinkSql.applyToApplication(application); + + assertThat(flinkSql.getSql()).isNull(); + assertThat(application.getFlinkSql()).isNull(); + } + @Test void shouldRejectCorruptPersistedSql() { FlinkSql flinkSql = flinkSql("not-compressed-sql"); diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/ResourceServiceTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/ResourceServiceTest.java index 91b6a547be..e6fbf23484 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/ResourceServiceTest.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/ResourceServiceTest.java @@ -18,10 +18,11 @@ package org.apache.streampark.console.core.service; import org.apache.streampark.console.SpringUnitTestBase; +import org.apache.streampark.console.base.util.WebUtils; +import org.apache.streampark.console.core.bean.UploadResponse; import org.apache.hc.core5.http.ContentType; -import org.h2.store.fs.FileUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.springframework.beans.factory.annotation.Autowired; @@ -31,8 +32,12 @@ import java.io.File; import java.nio.file.Files; import java.nio.file.Path; +import java.util.jar.Attributes; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; /** org.apache.streampark.console.core.service.ResourceServiceTest. */ class ResourceServiceTest extends SpringUnitTestBase { @@ -41,14 +46,15 @@ class ResourceServiceTest extends SpringUnitTestBase { private ResourceService resourceService; @Test - void testUpload(@TempDir Path tempDir) throws Exception { - // specify the file path - File fileToStoreUploadFile = - new File(tempDir.toFile().getAbsolutePath() + "/fileToStoreUploadFile"); - FileUtils.createFile(fileToStoreUploadFile.getAbsolutePath()); - + void uploadUsesGeneratedPath(@TempDir Path tempDir) throws Exception { File fileToUpload = new File(tempDir.toFile().getAbsolutePath() + "/fileToUpload.jar"); - FileUtils.createFile(fileToUpload.getAbsolutePath()); + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + try ( + JarOutputStream ignored = + new JarOutputStream(Files.newOutputStream(fileToUpload.toPath()), manifest)) { + // A manifest-only JAR is sufficient for upload validation. + } assertThat(fileToUpload).exists(); MultipartFile mulFile = new MockMultipartFile( @@ -56,7 +62,29 @@ void testUpload(@TempDir Path tempDir) throws Exception { fileToUpload.getAbsolutePath(), // originalFilename (eg: path + fileName = // /tmp/file/streampark.jar) ContentType.APPLICATION_OCTET_STREAM.toString(), - Files.newInputStream(fileToStoreUploadFile.toPath())); - resourceService.upload(mulFile); + Files.newInputStream(fileToUpload.toPath())); + UploadResponse response = resourceService.upload(mulFile); + Path uploaded = Path.of(response.getPath()); + try { + assertThat(uploaded).exists(); + assertThat(uploaded.getParent().toRealPath()) + .isEqualTo(WebUtils.getAppTempDir().toPath().toRealPath()); + assertThat(uploaded.getFileName().toString()) + .startsWith("upload-") + .endsWith(".jar") + .doesNotContain(fileToUpload.getName()); + } finally { + Files.deleteIfExists(uploaded); + } + } + + @Test + void rejectUnsupportedUpload() { + MultipartFile file = + new MockMultipartFile( + "file", "payload.exe", ContentType.APPLICATION_OCTET_STREAM.toString(), new byte[]{1}); + + assertThatThrownBy(() -> resourceService.upload(file)) + .hasMessageContaining("Only JAR and Python resources"); } } diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/impl/ProxyServiceImplTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/impl/ProxyServiceImplTest.java new file mode 100644 index 0000000000..0e4f9c0752 --- /dev/null +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/impl/ProxyServiceImplTest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.console.core.service.impl; + +import okhttp3.HttpUrl; +import org.junit.jupiter.api.Test; +import org.springframework.mock.web.MockHttpServletRequest; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Verifies that request data cannot alter proxy authority or escape its route. */ +class ProxyServiceImplTest { + + private final ProxyServiceImpl service = new ProxyServiceImpl(); + + @Test + void appendProxyPathAndQuery() { + MockHttpServletRequest request = + request("/proxy/flink/7/jobs/42", "state=RUNNING"); + + HttpUrl target = service.proxyUrl( + "http://cluster.internal:8081/base/", request, "/proxy/flink/7"); + + assertThat(target.host()).isEqualTo("cluster.internal"); + assertThat(target.port()).isEqualTo(8081); + assertThat(target.encodedPath()).isEqualTo("/base/jobs/42"); + assertThat(target.queryParameter("state")).isEqualTo("RUNNING"); + } + + @Test + void rejectProxyTraversal() { + String[] paths = { + "/proxy/flink/7/../admin", + "/proxy/flink/7/%2e%2e/admin", + "/proxy/flink/7/%252e%252e/admin" + }; + + for (String path : paths) { + MockHttpServletRequest request = request(path, null); + assertThatThrownBy( + () -> service.proxyUrl( + "http://cluster.internal:8081/", request, "/proxy/flink/7")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("traversal"); + } + } + + @Test + void rejectPathOutsideRoute() { + MockHttpServletRequest request = request("/proxy/flink/70/jobs", null); + + assertThatThrownBy( + () -> service.proxyUrl( + "http://cluster.internal:8081/", request, "/proxy/flink/7")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("outside the proxy route"); + } + + private static MockHttpServletRequest request(String path, String query) { + MockHttpServletRequest request = new MockHttpServletRequest("GET", path); + request.setQueryString(query); + return request; + } +} diff --git a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/YarnJarUploader.java b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/YarnJarUploader.java index 773c7bcf2b..768c8c0d11 100644 --- a/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/YarnJarUploader.java +++ b/streampark-flink/streampark-flink-packer/src/main/java/org/apache/streampark/flink/packer/pipeline/YarnJarUploader.java @@ -56,7 +56,7 @@ private static void uploadFile(FsOperator fsOperator, File originFile, String ta new FileInputStream(originFile), inputStream -> { try { - if (!DigestUtils.md5Hex(inputStream).equals(fsOperator.fileMd5(uploadFile))) { + if (!DigestUtils.sha256Hex(inputStream).equals(fsOperator.fileSha256(uploadFile))) { fsOperator.upload(originFile.getAbsolutePath(), uploadFile); } } catch (IOException e) { From cf129f30716b48781221c6315c290c693b94dc7a Mon Sep 17 00:00:00 2001 From: benjobs Date: Fri, 4 Sep 2026 00:15:49 +0800 Subject: [PATCH 3/9] [Build] Resolve remaining Sonar findings --- .../configuration/CommandLineParser.java | 22 ++-- .../common/configuration/ConfigOption.java | 5 +- .../configuration/ConfigValueTypes.java | 2 +- .../common/configuration/Configuration.java | 27 +++-- .../configuration/JvmOptionsParser.java | 7 +- .../streampark/common/fs/FsOperator.java | 12 -- .../streampark/common/fs/HdfsOperator.java | 11 -- .../streampark/common/fs/LfsOperator.java | 16 --- .../common/util/FlinkConfigurationUtils.java | 46 ++++---- .../streampark/common/util/HdfsUtils.java | 13 --- .../streampark/common/util/OkHttpUtils.java | 62 ++++++----- .../streampark/common/util/YamlParser.java | 103 +++++++++--------- .../configuration/ConfigurationTest.java | 21 ++++ .../streampark/common/fs/LfsOperatorTest.java | 45 ++++---- .../console/base/util/BashJavaUtils.java | 6 +- .../console/base/util/WebUtils.java | 45 +++++--- .../console/core/bean/MavenConfig.java | 10 +- .../console/core/runner/EnvInitializer.java | 2 +- .../core/service/impl/ProjectServiceImpl.java | 2 +- .../core/service/impl/ProxyServiceImpl.java | 8 +- .../core/util/FlinkShellCommandBuilder.java | 2 +- .../base/util/DependencyUtilsTest.java | 9 +- 22 files changed, 248 insertions(+), 228 deletions(-) diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineParser.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineParser.java index 1854187cde..30e7dd3e07 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineParser.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/CommandLineParser.java @@ -49,13 +49,10 @@ private CommandLineParser() { public static Configuration parse(String[] arguments) { Objects.requireNonNull(arguments, "arguments must not be null"); Map values = new LinkedHashMap<>(); - for (int index = 0; index < arguments.length; index++) { + int index = 0; + while (index < arguments.length) { String token = Objects.requireNonNull(arguments[index], "argument must not be null"); - if (!token.startsWith(LONG_OPTION_PREFIX) || token.length() == LONG_OPTION_PREFIX.length()) { - throw new ConfigException("Expected a --key argument but found '" + token + "'"); - } - - String expression = token.substring(LONG_OPTION_PREFIX.length()); + String expression = optionExpression(token); int separator = expression.indexOf('='); String key; Object value; @@ -64,13 +61,16 @@ public static Configuration parse(String[] arguments) { if (separator >= 0) { key = expression.substring(0, separator); value = expression.substring(separator + 1); + index++; } else { key = expression; if (index + 1 < arguments.length && !arguments[index + 1].startsWith(LONG_OPTION_PREFIX)) { - value = arguments[++index]; + value = arguments[index + 1]; + index += 2; } else { value = true; + index++; } } if (key.trim().isEmpty()) { @@ -84,4 +84,12 @@ public static Configuration parse(String[] arguments) { .add("command line", ConfigSource.COMMAND_LINE, values) .build(); } + + private static String optionExpression(String token) { + if (!token.startsWith(LONG_OPTION_PREFIX) + || token.length() == LONG_OPTION_PREFIX.length()) { + throw new ConfigException("Expected a --key argument but found '" + token + "'"); + } + return token.substring(LONG_OPTION_PREFIX.length()); + } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOption.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOption.java index 3220d3b9ef..de68d604d5 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOption.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigOption.java @@ -17,7 +17,6 @@ package org.apache.streampark.common.configuration; -import java.io.Serializable; import java.util.List; import java.util.Objects; import java.util.function.Predicate; @@ -30,9 +29,7 @@ * * @param value type */ -public final class ConfigOption implements Serializable { - - private static final long serialVersionUID = 1L; +public final class ConfigOption { private final String key; private final ConfigValueType valueType; diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigValueTypes.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigValueTypes.java index 272cd8569c..6eb9d91fb3 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigValueTypes.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/ConfigValueTypes.java @@ -128,7 +128,7 @@ private static Long toLong(Object value) { long longValue = number.longValue(); // Number#longValue silently truncates floating-point input. Comparing the round-trip // value rejects that lossy coercion before it reaches an integral option. - if (Double.compare(number.doubleValue(), (double) longValue) != 0) { + if (Double.compare(number.doubleValue(), longValue) != 0) { throw new IllegalArgumentException("expected an integral number"); } return longValue; diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java index 587c71e17a..96cb05b30d 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java @@ -40,6 +40,8 @@ public final class Configuration implements ReadableConfig, Serializable { private static final long serialVersionUID = 1L; + private static final String NULL_KEY = "key must not be null"; + private static final String NULL_OPTION = "option must not be null"; private static final Configuration EMPTY = new Configuration(Collections.emptyMap()); private final Map values; @@ -90,7 +92,7 @@ public T get(ConfigOption option) { @Override public Optional getOptional(ConfigOption option) { - Objects.requireNonNull(option, "option must not be null"); + Objects.requireNonNull(option, NULL_OPTION); // The canonical key always wins over fallbacks. Fallbacks are declaration-time migration // metadata and never copied into the effective snapshot. String resolvedKey = resolveKey(option); @@ -118,7 +120,7 @@ public Optional getOptional(ConfigOption option) { @Override public boolean contains(ConfigOption option) { - return resolveKey(Objects.requireNonNull(option, "option must not be null")) != null; + return resolveKey(Objects.requireNonNull(option, NULL_OPTION)) != null; } /** @@ -130,7 +132,7 @@ public boolean contains(ConfigOption option) { * @return whether the key is present */ public boolean containsKey(String key) { - return values.containsKey(Objects.requireNonNull(key, "key must not be null")); + return values.containsKey(Objects.requireNonNull(key, NULL_KEY)); } /** @@ -152,7 +154,7 @@ public String getString(String key) { * @return canonical string representation, or an empty optional when absent */ public Optional getOptionalString(String key) { - Value value = values.get(Objects.requireNonNull(key, "key must not be null")); + Value value = values.get(Objects.requireNonNull(key, NULL_KEY)); return value == null ? Optional.empty() : Optional.of(ConfigValueTypes.formatUnknown(value.rawValue)); @@ -165,7 +167,7 @@ public Optional getOptionalString(String key) { * @return effective value origin, or an empty optional when absent */ public Optional origin(String key) { - Value value = values.get(Objects.requireNonNull(key, "key must not be null")); + Value value = values.get(Objects.requireNonNull(key, NULL_KEY)); return value == null ? Optional.empty() : Optional.of(value.origin); } @@ -289,6 +291,17 @@ static T immutableValue(T value) { return value; } + private static Serializable serializableValue(Object value) { + Object immutable = immutableValue(value); + if (!(immutable instanceof Serializable)) { + throw new ConfigException( + "Configuration value of type " + + immutable.getClass().getName() + + " is not serializable"); + } + return (Serializable) immutable; + } + @Override public String toString() { return toRedactedMap().toString(); @@ -298,11 +311,11 @@ private static final class Value implements Serializable { private static final long serialVersionUID = 1L; - private final Object rawValue; + private final Serializable rawValue; private final ConfigOrigin origin; private Value(Object rawValue, ConfigOrigin origin) { - this.rawValue = immutableValue(rawValue); + this.rawValue = serializableValue(rawValue); this.origin = origin; } } diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/JvmOptionsParser.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/JvmOptionsParser.java index 45dc47081c..dd6cca43df 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/configuration/JvmOptionsParser.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/JvmOptionsParser.java @@ -45,11 +45,13 @@ private JvmOptionsParser() { public static Configuration parse(String options) { List tokens = CommandLineTokenizer.tokenize(options); Map properties = new LinkedHashMap<>(); - for (int index = 0; index < tokens.size(); index++) { + int index = 0; + while (index < tokens.size()) { String token = tokens.get(index); String expression; if (PREFIX.equals(token)) { - if (++index >= tokens.size()) { + index++; + if (index >= tokens.size()) { throw new ConfigException("-D must be followed by key=value"); } expression = tokens.get(index); @@ -68,6 +70,7 @@ public static Configuration parse(String options) { if (properties.putIfAbsent(key, value) != null) { throw new ConfigException("Duplicate JVM property: " + key); } + index++; } return Configuration.builder() .add("JVM options", ConfigSource.COMMAND_LINE, properties) diff --git a/streampark-common/src/main/java/org/apache/streampark/common/fs/FsOperator.java b/streampark-common/src/main/java/org/apache/streampark/common/fs/FsOperator.java index e5f7978d73..35f47e78dd 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/fs/FsOperator.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/fs/FsOperator.java @@ -65,11 +65,6 @@ public void move(String srcPath, String dstPath) { LfsOperator.move(srcPath, dstPath); } - @Override - public String fileMd5(String path) { - return LfsOperator.fileMd5(path); - } - @Override public String fileSha256(String path) { return LfsOperator.fileSha256(path); @@ -119,11 +114,6 @@ public void move(String srcPath, String dstPath) { HdfsOperator.move(srcPath, dstPath); } - @Override - public String fileMd5(String path) { - return HdfsOperator.fileMd5(path); - } - @Override public String fileSha256(String path) { return HdfsOperator.fileSha256(path); @@ -186,8 +176,6 @@ public void copyDir(String srcPath, String dstPath) { public abstract void move(String srcPath, String dstPath); - public abstract String fileMd5(String path); - /** * Returns the SHA-256 digest of a file for collision-resistant content comparison. * diff --git a/streampark-common/src/main/java/org/apache/streampark/common/fs/HdfsOperator.java b/streampark-common/src/main/java/org/apache/streampark/common/fs/HdfsOperator.java index 7489e2d285..e6660b7cdc 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/fs/HdfsOperator.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/fs/HdfsOperator.java @@ -116,17 +116,6 @@ public static void mkCleanDirs(String path) { mkdirs(path); } - public static String fileMd5(String path) { - if (path == null || path.isEmpty()) { - throw new IllegalArgumentException("[StreamPark] HdfsOperator.fileMd5: file must not be null."); - } - try { - return HdfsUtils.fileMd5(toHdfsPath(path)); - } catch (IOException e) { - throw new IllegalStateException("[StreamPark] Failed to compute md5 for: " + path, e); - } - } - /** * Returns the SHA-256 digest of an HDFS file. * diff --git a/streampark-common/src/main/java/org/apache/streampark/common/fs/LfsOperator.java b/streampark-common/src/main/java/org/apache/streampark/common/fs/LfsOperator.java index bba4318308..b4b9fc4f4d 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/fs/LfsOperator.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/fs/LfsOperator.java @@ -21,7 +21,6 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import java.io.File; @@ -184,21 +183,6 @@ public static void copyDir(String srcPath, String dstPath, boolean delSrc, boole } } - public static String fileMd5(String path) { - if (path == null || path.isEmpty()) { - throw new IllegalArgumentException("[StreamPark] LFsOperator.fileMd5: file must not be null."); - } - File file = new File(path); - if (!file.exists()) { - throw new IllegalArgumentException("[StreamPark] LFsOperator.fileMd5: file must exists."); - } - try (FileInputStream inputStream = new FileInputStream(path)) { - return DigestUtils.md5Hex(IOUtils.toByteArray(inputStream)); - } catch (IOException e) { - throw new IllegalStateException("[StreamPark] Failed to compute md5 for: " + path, e); - } - } - /** * Returns the SHA-256 digest of a local file. * diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java index 282a2fa043..1bac3449e9 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java @@ -286,31 +286,37 @@ private static Map loadLegacy(BufferedReader reader, String reso lineNumber++; int commentStart = line.indexOf('#'); String content = (commentStart < 0 ? line : line.substring(0, commentStart)).trim(); - if (content.isEmpty()) { - continue; + if (!content.isEmpty()) { + parseLegacyLine(configuration, content, resourceName, lineNumber); } - String[] entry = content.split(": ", 2); - if (entry.length != 2) { - LOG.warn( - "Ignoring invalid Flink configuration line {}:{}; expected 'key: value'", - resourceName, - lineNumber); - continue; - } - String key = entry[0].trim(); - String value = entry[1].trim(); - if (key.isEmpty() || value.isEmpty()) { - LOG.warn( - "Ignoring empty Flink configuration entry at {}:{}", - resourceName, - lineNumber); - continue; - } - configuration.put(key, value); } return configuration; } + /** Parses one non-empty legacy Flink configuration line. */ + private static void parseLegacyLine( + Map configuration, + String content, + String resourceName, + int lineNumber) { + int separator = content.indexOf(": "); + if (separator < 0) { + LOG.warn( + "Ignoring invalid Flink configuration line {}:{}; expected 'key: value'", + resourceName, + lineNumber); + return; + } + String key = content.substring(0, separator).trim(); + String value = content.substring(separator + 2).trim(); + if (key.isEmpty() || value.isEmpty()) { + LOG.warn( + "Ignoring empty Flink configuration entry at {}:{}", resourceName, lineNumber); + return; + } + configuration.put(key, value); + } + private static void logConfiguration(String action, Map configuration) { LOG.info("{} {} Flink configuration properties", action, configuration.size()); configuration.forEach( diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/HdfsUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/HdfsUtils.java index fbf581bd07..7edac08a48 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/HdfsUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/HdfsUtils.java @@ -168,19 +168,6 @@ public static void delete(String src) throws IOException { } } - public static String fileMd5(String fileName) throws IOException { - Path path = getPath(fileName); - FSDataInputStream in = HadoopUtils.hdfs().open(path); - try { - // MD5 is used for non-cryptographic file integrity checks only. - @SuppressWarnings("java:S4790") - String digest = DigestUtils.md5Hex(in); - return digest; - } finally { - in.close(); - } - } - /** * Returns the SHA-256 digest of an HDFS file. * diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/OkHttpUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/OkHttpUtils.java index 0329d0580a..43dabf5006 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/OkHttpUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/OkHttpUtils.java @@ -86,38 +86,44 @@ private static OkHttpClient createClient() { /** Retries transient failures only for methods whose requests are safe to replay. */ private static Interceptor newRetryInterceptor() { - return chain -> { - Request request = chain.request(); - if (!isRetryableMethod(request.method())) { - return chain.proceed(request); - } + return OkHttpUtils::retry; + } - Response response = null; - for (int attempt = 1; attempt <= MAX_RETRY_ATTEMPTS; attempt++) { - try { - response = chain.proceed(request); - if (response.isSuccessful() - || !isRetryableStatus(response.code()) - || attempt == MAX_RETRY_ATTEMPTS) { - return response; - } - response.close(); - response = null; - } catch (IOException e) { - if (attempt == MAX_RETRY_ATTEMPTS) { - throw e; - } + private static Response retry(Interceptor.Chain chain) throws IOException { + Request request = chain.request(); + if (!isRetryableMethod(request.method())) { + return chain.proceed(request); + } + for (int attempt = 1; attempt <= MAX_RETRY_ATTEMPTS; attempt++) { + try { + Response response = chain.proceed(request); + if (!shouldRetry(response, attempt)) { + return response; } - - try { - Thread.sleep((1L << attempt) * 1000L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IOException("Interrupted while retrying HTTP request", e); + response.close(); + } catch (IOException e) { + if (attempt == MAX_RETRY_ATTEMPTS) { + throw e; } } - throw new IOException("HTTP retry loop ended without a response"); - }; + pause(attempt); + } + throw new IOException("HTTP retry loop ended without a response"); + } + + private static boolean shouldRetry(Response response, int attempt) { + return !response.isSuccessful() + && isRetryableStatus(response.code()) + && attempt < MAX_RETRY_ATTEMPTS; + } + + private static void pause(int attempt) throws IOException { + try { + Thread.sleep((1L << attempt) * 1000L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException("Interrupted while retrying HTTP request", e); + } } private static boolean isRetryableMethod(String method) { diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java b/streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java index 23cd3eff40..b41d42c4f4 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/YamlParser.java @@ -222,61 +222,68 @@ private static String foldLines(String text) { String lineSeparator = text.contains("\r\n") ? "\r\n" : "\n"; String[] lines = text.split("\\r?\\n", -1); StringBuilder normalized = new StringBuilder(text.length()); - for (int index = 0; index < lines.length; index++) { + int index = 0; + while (index < lines.length) { String currentLine = lines[index]; Matcher matcher = EMPTY_VALUE_PROPERTY.matcher(currentLine); - if (!matcher.matches()) { + FoldBlock block = + matcher.matches() ? findFoldBlock(lines, index, indent(currentLine)) : null; + if (block == null) { appendLine(normalized, currentLine, lineSeparator, index < lines.length - 1); - continue; + index++; + } else { + normalized + .append(matcher.group(1)) + .append(matcher.group(2).trim()) + .append(": ") + .append(String.join(" ", block.entries)); + if (block.end < lines.length) { + normalized.append(lineSeparator); + } + index = block.end; } + } + return normalized.toString(); + } - int currentIndent = indent(currentLine); - int nextIndex = index + 1; - List scalarEntries = new ArrayList<>(); - boolean structuredChild = false; - - // A property without an inline value may either own nested YAML or contain plain text - // entered on following lines. Inspect the entire indented region before modifying it. - while (nextIndex < lines.length) { - String childLine = lines[nextIndex]; - if (isBlank(childLine)) { - nextIndex++; - continue; - } - if (indent(childLine) <= currentIndent) { - break; - } - String trimmedChild = childLine.trim(); - if (trimmedChild.startsWith("#")) { - nextIndex++; - continue; - } - if (isStructured(trimmedChild)) { - // Never partially fold a mapping or sequence. Leaving the original block intact - // allows the strict parser to retain its structure on the next attempt. - structuredChild = true; - break; + /** Finds plain indented lines that may be folded into a scalar property. */ + private static FoldBlock findFoldBlock(String[] lines, int index, int parentIndent) { + int next = index + 1; + List entries = new ArrayList<>(); + // A property without an inline value may own nested YAML or editor-style plain text. The + // entire indented region is inspected before any line is modified. + while (next < lines.length) { + String child = lines[next]; + if (isBlank(child)) { + next++; + } else if (indent(child) <= parentIndent) { + break; + } else { + String trimmed = child.trim(); + if (trimmed.startsWith("#")) { + next++; + } else if (isStructured(trimmed)) { + // Nested mappings and sequences must remain untouched for the strict parser. + return null; + } else { + entries.add(trimmed); + next++; } - scalarEntries.add(trimmedChild); - nextIndex++; } + } + return entries.isEmpty() ? null : new FoldBlock(next, entries); + } - if (scalarEntries.isEmpty() || structuredChild) { - appendLine(normalized, currentLine, lineSeparator, index < lines.length - 1); - continue; - } + /** Immutable description of a scalar continuation region. */ + private static final class FoldBlock { - normalized - .append(matcher.group(1)) - .append(matcher.group(2).trim()) - .append(": ") - .append(String.join(" ", scalarEntries)); - if (nextIndex < lines.length) { - normalized.append(lineSeparator); - } - index = nextIndex - 1; + private final int end; + private final List entries; + + private FoldBlock(int end, List entries) { + this.end = end; + this.entries = entries; } - return normalized.toString(); } private static Map parseBytes(byte[] bytes) { @@ -311,10 +318,8 @@ private static void flatten(Map source, String prefix, Map Object value = entry.getValue(); if (value instanceof Map) { flatten((Map) value, key, target); - } else if (value != null) { - if (target.putIfAbsent(key, value) != null) { - throw new ConfigException("Duplicate configuration key: " + key); - } + } else if (value != null && target.putIfAbsent(key, value) != null) { + throw new ConfigException("Duplicate configuration key: " + key); } } } diff --git a/streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationTest.java b/streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationTest.java index 8070c7b2bb..0ef26e88bc 100644 --- a/streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationTest.java +++ b/streampark-common/src/test/java/org/apache/streampark/common/configuration/ConfigurationTest.java @@ -19,6 +19,8 @@ import org.junit.jupiter.api.Test; +import java.io.ByteArrayOutputStream; +import java.io.ObjectOutputStream; import java.time.Duration; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -211,6 +213,25 @@ void copyMutableValues() { assertThat(loader.load().get(hosts)).containsExactly("pending-a"); } + @Test + void enforceSerializableValues() throws Exception { + Map invalid = Collections.singletonMap("invalid", new Object()); + assertThatThrownBy( + () -> Configuration.builder().add("test", ConfigSource.FILE, invalid)) + .isInstanceOf(ConfigException.class) + .hasMessageContaining("not serializable"); + + Configuration configuration = + Configuration.builder() + .add("test", ConfigSource.FILE, Map.of("name", "orders")) + .build(); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (ObjectOutputStream output = new ObjectOutputStream(bytes)) { + output.writeObject(configuration); + } + assertThat(bytes.size()).isPositive(); + } + @Test void rejectInvalidAndMissingValues() { Configuration invalid = diff --git a/streampark-common/src/test/java/org/apache/streampark/common/fs/LfsOperatorTest.java b/streampark-common/src/test/java/org/apache/streampark/common/fs/LfsOperatorTest.java index 029bd06fea..319d180a1d 100644 --- a/streampark-common/src/test/java/org/apache/streampark/common/fs/LfsOperatorTest.java +++ b/streampark-common/src/test/java/org/apache/streampark/common/fs/LfsOperatorTest.java @@ -173,9 +173,9 @@ void testDelete() throws Exception { }); } - private String md5Hex(File file) throws Exception { + private String sha256Hex(File file) throws Exception { try (FileInputStream input = new FileInputStream(file)) { - return DigestUtils.md5Hex(IOUtils.toByteArray(input)); + return DigestUtils.sha256Hex(IOUtils.toByteArray(input)); } } @@ -185,16 +185,16 @@ private boolean sameFilesHex(File[] f1, File[] f2) { if (!names1.equals(names2)) { return false; } - List md5s1 = - Arrays.stream(f1).map(this::md5HexSafe).sorted().collect(Collectors.toList()); - List md5s2 = - Arrays.stream(f2).map(this::md5HexSafe).sorted().collect(Collectors.toList()); - return md5s1.equals(md5s2); + List digests1 = + Arrays.stream(f1).map(this::sha256HexSafe).sorted().collect(Collectors.toList()); + List digests2 = + Arrays.stream(f2).map(this::sha256HexSafe).sorted().collect(Collectors.toList()); + return digests1.equals(digests2); } - private String md5HexSafe(File file) { + private String sha256HexSafe(File file) { try { - return md5Hex(file); + return sha256Hex(file); } catch (Exception e) { throw new RuntimeException(e); } @@ -259,21 +259,21 @@ void testCopy() throws Exception { assertDoesNotThrow( () -> { File out = genRandomFile(outputDir + "/out-9", "114514-233.dat"); - String md5Before = md5Hex(out); + String digestBefore = sha256Hex(out); LfsOperator.copy(file.getAbsolutePath(), out.getAbsolutePath(), false, true); - String md5After = md5Hex(new File(out.getAbsolutePath())); - assertNotEquals(md5Before, md5After); - assertEquals(md5After, md5Hex(file)); + String digestAfter = sha256Hex(new File(out.getAbsolutePath())); + assertNotEquals(digestBefore, digestAfter); + assertEquals(digestAfter, sha256Hex(file)); }); assertDoesNotThrow( () -> { File out = genRandomFile(outputDir + "/out-10", "114514-233.dat"); - String md5Before = md5Hex(out); + String digestBefore = sha256Hex(out); LfsOperator.copy(file.getAbsolutePath(), out.getAbsolutePath(), false, false); - String md5After = md5Hex(new File(out.getAbsolutePath())); - assertEquals(md5Before, md5After); - assertNotEquals(md5After, md5Hex(file)); + String digestAfter = sha256Hex(new File(out.getAbsolutePath())); + assertEquals(digestBefore, digestAfter); + assertNotEquals(digestAfter, sha256Hex(file)); }); }); } @@ -333,9 +333,9 @@ void testMove() throws Exception { assertDoesNotThrow( () -> { File sourceFile = genRandomFile(outputDir); - String sourceMd5; + String sourceDigest; try (FileInputStream input = new FileInputStream(sourceFile)) { - sourceMd5 = DigestUtils.md5Hex(IOUtils.toByteArray(input)); + sourceDigest = DigestUtils.sha256Hex(IOUtils.toByteArray(input)); } String targetPath = outputDir + "/target-1"; LfsOperator.move(sourceFile.getAbsolutePath(), targetPath); @@ -343,7 +343,9 @@ void testMove() throws Exception { File targetFile = new File(targetPath, sourceFile.getName()); assertTrue(targetFile.exists()); try (FileInputStream input = new FileInputStream(targetFile)) { - assertEquals(sourceMd5, DigestUtils.md5Hex(IOUtils.toByteArray(input))); + assertEquals( + sourceDigest, + DigestUtils.sha256Hex(IOUtils.toByteArray(input))); } }); @@ -396,9 +398,6 @@ void testMove() throws Exception { @Test void rejectInvalidDigestPaths() { - assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileMd5(null)); - assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileMd5("")); - assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileMd5("ttt/144514.dat")); assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileSha256(null)); assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileSha256("")); assertThrows(IllegalArgumentException.class, () -> LfsOperator.fileSha256("ttt/144514.dat")); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java index 5b7dfab152..b41e0a31d5 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java @@ -55,8 +55,8 @@ public static void main(String[] args) throws IOException { System.out.println(value); break; case "--check_port": - int port = Integer.parseInt(actionArgs[0]); - try (Socket ignored = new Socket(localhost, port)) { + int checkedPort = Integer.parseInt(actionArgs[0]); + try (Socket ignored = new Socket(localhost, checkedPort)) { System.out.println("used"); } catch (Exception e) { System.out.println("free"); @@ -64,7 +64,7 @@ public static void main(String[] args) throws IOException { break; case "--free_port": int start = Integer.parseInt(actionArgs[0]); - for (port = start; port < 65535; port++) { + for (int port = start; port < 65535; port++) { try (Socket ignored = new Socket(localhost, port)) { } catch (Exception e) { System.out.println(port); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java index 371a0682a7..8883de5c50 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/WebUtils.java @@ -146,6 +146,14 @@ public static void http( HttpUrl url, HttpServletRequest request, HttpServletResponse response) throws IOException { + Request upstreamRequest = buildProxyRequest(url, request, requestHeaders(request)); + try (Response upstream = OkHttpUtils.call(upstreamRequest)) { + writeProxyResponse(upstream, response); + } + } + + /** Copies end-to-end request headers while excluding proxy and browser security metadata. */ + private static Headers requestHeaders(HttpServletRequest request) { Headers.Builder headersBuilder = new Headers.Builder(); Set skippedHeaders = skippedHeaders(request.getHeader("Connection"), REQUEST_SKIP_HEADERS); @@ -162,20 +170,32 @@ public static void http( continue; } Enumeration values = request.getHeaders(headerName); - if (values == null) { - continue; - } - while (values.hasMoreElements()) { - String value = values.nextElement(); - if (value != null) { - headersBuilder.add(headerName, value); - } + if (values != null) { + addHeaderValues(headersBuilder, headerName, values); } } } + return headersBuilder.build(); + } - Request.Builder requestBuilder = - new Request.Builder().url(url).headers(headersBuilder.build()); + private static void addHeaderValues( + Headers.Builder headers, + String name, + Enumeration values) { + while (values.hasMoreElements()) { + String value = values.nextElement(); + if (value != null) { + headers.add(name, value); + } + } + } + + /** Creates the replayable OkHttp request used by the shared client. */ + private static Request buildProxyRequest( + HttpUrl url, + HttpServletRequest request, + Headers headers) throws IOException { + Request.Builder requestBuilder = new Request.Builder().url(url).headers(headers); // OkHttp derives Host and Content-Length from the upstream URL and replayable request body. String method = request.getMethod(); if (HttpMethod.GET.matches(method) || HttpMethod.HEAD.matches(method)) { @@ -188,10 +208,7 @@ public static void http( : null; requestBuilder.method(method, RequestBody.create(content, contentType)); } - - try (Response upstream = OkHttpUtils.call(requestBuilder.build())) { - writeProxyResponse(upstream, response); - } + return requestBuilder.build(); } private static void writeProxyResponse( diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/MavenConfig.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/MavenConfig.java index d0fc91b412..fe134132b8 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/MavenConfig.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/MavenConfig.java @@ -37,6 +37,8 @@ @Setter public class MavenConfig { + private static final String DATABASE_SETTING = "database setting"; + /** File path for Maven settings. */ private String mvnSettings; @@ -84,19 +86,19 @@ public static MavenConfig fromSetting() { public void updateConfig() { if (StringUtils.isNotBlank(mvnSettings)) { - GlobalConfiguration.set(MavenOptions.SETTINGS_PATH, mvnSettings, "database setting"); + GlobalConfiguration.set(MavenOptions.SETTINGS_PATH, mvnSettings, DATABASE_SETTING); } if (StringUtils.isNotBlank(mvnRepository)) { - GlobalConfiguration.set(MavenOptions.REPOSITORY_URL, mvnRepository, "database setting"); + GlobalConfiguration.set(MavenOptions.REPOSITORY_URL, mvnRepository, DATABASE_SETTING); } if (StringUtils.isNotBlank(mvnAuthUser)) { - GlobalConfiguration.set(MavenOptions.USER_NAME, mvnAuthUser, "database setting"); + GlobalConfiguration.set(MavenOptions.USER_NAME, mvnAuthUser, DATABASE_SETTING); } if (StringUtils.isNotBlank(mvnAuthPassword)) { - GlobalConfiguration.set(MavenOptions.PASSWORD, mvnAuthPassword, "database setting"); + GlobalConfiguration.set(MavenOptions.PASSWORD, mvnAuthPassword, DATABASE_SETTING); } } } diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java index d009032ef1..9163447f17 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java @@ -70,7 +70,7 @@ public class EnvInitializer implements ApplicationRunner { private final FileFilter fileFilter = p -> !".gitkeep".equals(p.getName()); private static final Pattern PATTERN_FLINK_SHIMS_JAR = Pattern.compile( - "^streampark-flink-shims_flink-(1\\.1[8-9]|1\\.2[0-9]|2\\.[0-3])-(.*).jar$", + "^streampark-flink-shims_flink-(1\\.1[8-9]|1\\.2\\d|2\\.[0-3])-(.*).jar$", Pattern.CASE_INSENSITIVE | Pattern.DOTALL); /** diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java index 1114607fab..15a37b211f 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java @@ -325,7 +325,7 @@ public List> listConf(Project project) { } catch (Exception e) { log.error("List project conf failed", e); } - return null; + return Collections.emptyList(); } /** Resolves a request-supplied module below the project's distribution directory. */ diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java index ced07f2589..84c3d410c9 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProxyServiceImpl.java @@ -54,6 +54,8 @@ @Service public class ProxyServiceImpl implements ProxyService { + private static final String PROXY_PATH = "/proxy/"; + @Autowired private FlinkClusterService flinkClusterService; @@ -73,7 +75,7 @@ public void proxyFlink( case YARN_PER_JOB: case YARN_APPLICATION: case YARN_SESSION: - url = YarnUtils.getRMWebAppProxyURL() + "/proxy/" + app.getClusterId(); + url = YarnUtils.getRMWebAppProxyURL() + PROXY_PATH + app.getClusterId(); proxyYarnRequest( request, response, @@ -110,7 +112,7 @@ public void proxySpark( switch (app.getDeployModeEnum()) { case YARN_CLIENT: case YARN_CLUSTER: - String url = YarnUtils.getRMWebAppProxyURL() + "/proxy/" + app.getClusterId(); + String url = YarnUtils.getRMWebAppProxyURL() + PROXY_PATH + app.getClusterId(); proxyYarnRequest( request, response, @@ -132,7 +134,7 @@ public void proxyYarn( unavailableResponse(response, "The yarn application id is null."); return; } - String url = YarnUtils.getRMWebAppProxyURL() + "/proxy/" + yarnId + "/"; + String url = YarnUtils.getRMWebAppProxyURL() + PROXY_PATH + yarnId + "/"; proxyYarnRequest( request, response, diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilder.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilder.java index c6ce7602a5..a33d35cefa 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilder.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkShellCommandBuilder.java @@ -226,7 +226,7 @@ private static void mergeProgramArgs(Map optionMap, String[] arg org.apache.commons.cli.CommandLine line = PARSER.parse(FLINK_OPTIONS, args, false); for (org.apache.commons.cli.Option option : line.getOptions()) { String key = "-" + option.getLongOpt().trim(); - optionMap.put(key, option.hasArg() ? option.getValue() : true); + optionMap.put(key, option.hasArg() ? option.getValue() : Boolean.TRUE); } } catch (org.apache.commons.cli.ParseException e) { throw new ConfigException("Invalid program launcher arguments", e); diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/DependencyUtilsTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/DependencyUtilsTest.java index 434c313bef..735382a550 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/DependencyUtilsTest.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/base/util/DependencyUtilsTest.java @@ -52,7 +52,7 @@ class DependencyUtilsTest { @Disabled("Disabled due to unstable performance.") @Test - public void resolveFlinkConnector() throws Exception { + void resolveFlinkConnector() throws Exception { Artifact artifact = new Artifact("com.ververica", "flink-connector-mysql-cdc", "2.4.1", null); @@ -125,13 +125,6 @@ private String getOptionDefaultValue(ConfigOption option) { return value.toString(); } - @Test - public void testDuration() { - String s = "PT30H"; - Duration duration = Duration.parse(s); - System.out.println(duration.getSeconds()); - } - private List getConnectorFactory(File connector) throws Exception { String configFile = "META-INF/services/org.apache.flink.table.factories.Factory"; JarFile jarFile = new JarFile(connector); From a190ef417af274fd59eaae6e9500d19ac5bee3af Mon Sep 17 00:00:00 2001 From: benjobs Date: Fri, 4 Sep 2026 00:22:02 +0800 Subject: [PATCH 4/9] [Common] Reuse configuration null checks --- .../apache/streampark/common/configuration/Configuration.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java b/streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java index 96cb05b30d..ebd973d00d 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/configuration/Configuration.java @@ -378,7 +378,7 @@ public Builder add(Configuration configuration) { * @throws ConfigException when the value violates the option constraint */ public Builder set(ConfigOption option, T value, String originName) { - Objects.requireNonNull(option, "option must not be null").validate(value); + Objects.requireNonNull(option, NULL_OPTION).validate(value); mergeValue( option.key(), new Value(value, ConfigOrigin.of(ConfigSource.RUNTIME, originName))); @@ -392,7 +392,7 @@ public Builder set(ConfigOption option, T value, String originName) { * @return this builder */ public Builder remove(String key) { - values.remove(Objects.requireNonNull(key, "key must not be null")); + values.remove(Objects.requireNonNull(key, NULL_KEY)); return this; } From fbb23ad3861eab66711c72204b202a020abb9fb3 Mon Sep 17 00:00:00 2001 From: benjobs Date: Fri, 4 Sep 2026 00:39:07 +0800 Subject: [PATCH 5/9] [Flink] Fix cluster metric comparison --- .../FlinkApplicationActionServiceImpl.java | 18 ++++---- .../SparkApplicationActionServiceImpl.java | 21 +++++---- .../client/impl/AbstractFlinkClient.java | 2 +- .../flink/kubernetes/model/FlinkMetricCV.java | 4 +- .../kubernetes/model/FlinkMetricCVTest.java | 43 +++++++++++++++++++ 5 files changed, 68 insertions(+), 20 deletions(-) create mode 100644 streampark-flink/streampark-flink-kubernetes/src/test/java/org/apache/streampark/flink/kubernetes/model/FlinkMetricCVTest.java diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java index 414db6c7a4..32ae0d0fed 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/FlinkApplicationActionServiceImpl.java @@ -703,20 +703,20 @@ private Tuple2 getUserJarAndAppConf( break; case PYFLINK: - Resource resource = + Resource pythonResource = resourceService.findByResourceName(application.getTeamId(), application.getJar()); ApiAlertException.throwIfNull( - resource, "pyflink file can't be null, start application failed."); + pythonResource, "pyflink file can't be null, start application failed."); ApiAlertException.throwIfNull( - resource.getFilePath(), "pyflink file can't be null, start application failed."); + pythonResource.getFilePath(), "pyflink file can't be null, start application failed."); ApiAlertException.throwIfFalse( - resource.getFilePath().endsWith(Constants.PYTHON_SUFFIX), + pythonResource.getFilePath().endsWith(Constants.PYTHON_SUFFIX), "pyflink format error, must be a \".py\" suffix, start application failed."); - flinkUserJar = resource.getFilePath(); + flinkUserJar = pythonResource.getFilePath(); break; case FLINK_JAR: @@ -762,14 +762,16 @@ private Tuple2 getUserJarAndAppConf( case APACHE_FLINK: flinkUserJar = String.format("%s/%s", application.getAppHome(), application.getJar()); if (!FsOperator.hdfs().exists(flinkUserJar)) { - resource = + Resource jarResource = resourceService.findByResourceName( application.getTeamId(), application.getJar()); - if (resource != null && StringUtils.isNotBlank(resource.getFilePath())) { + if (jarResource != null + && StringUtils.isNotBlank(jarResource.getFilePath())) { flinkUserJar = String.format( "%s/%s", - application.getAppHome(), new File(resource.getFilePath()).getName()); + application.getAppHome(), + new File(jarResource.getFilePath()).getName()); } } break; diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationActionServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationActionServiceImpl.java index ddf3d5c4fd..c771c281c7 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationActionServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/application/impl/SparkApplicationActionServiceImpl.java @@ -442,19 +442,20 @@ private Tuple2 getUserJarAndAppConf( appConf = applicationConfig == null ? null : String.format("yaml://%s", applicationConfig.getContent()); - Resource resource = resourceService.findByResourceName(application.getTeamId(), application.getJar()); + Resource pythonResource = + resourceService.findByResourceName(application.getTeamId(), application.getJar()); ApiAlertException.throwIfNull( - resource, "pyflink file can't be null, start application failed."); + pythonResource, "pyflink file can't be null, start application failed."); ApiAlertException.throwIfNull( - resource.getFilePath(), "pyflink file can't be null, start application failed."); + pythonResource.getFilePath(), "pyflink file can't be null, start application failed."); ApiAlertException.throwIfFalse( - resource.getFilePath().endsWith(Constants.PYTHON_SUFFIX), + pythonResource.getFilePath().endsWith(Constants.PYTHON_SUFFIX), "pyflink format error, must be a \".py\" suffix, start application failed."); - sparkUserJar = resource.getFilePath(); + sparkUserJar = pythonResource.getFilePath(); break; case SPARK_JAR: @@ -496,13 +497,15 @@ private Tuple2 getUserJarAndAppConf( case APACHE_SPARK: sparkUserJar = String.format("%s/%s", application.getAppHome(), application.getJar()); if (!FsOperator.hdfs().exists(sparkUserJar)) { - resource = resourceService.findByResourceName( - application.getTeamId(), application.getJar()); - if (resource != null && StringUtils.isNotBlank(resource.getFilePath())) { + Resource jarResource = + resourceService.findByResourceName( + application.getTeamId(), application.getJar()); + if (jarResource != null + && StringUtils.isNotBlank(jarResource.getFilePath())) { sparkUserJar = String.format( "%s/%s", application.getAppHome(), - new File(resource.getFilePath()).getName()); + new File(jarResource.getFilePath()).getName()); } } break; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java index 9b2940e76e..bf2f0fe312 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java @@ -106,7 +106,7 @@ protected static FlinkException asFlinkException(Throwable throwable) { return (FlinkException) throwable; } if (throwable instanceof Exception) { - return new FlinkException((Exception) throwable); + return new FlinkException(throwable); } return new FlinkException(throwable.getMessage(), throwable); } diff --git a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/FlinkMetricCV.java b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/FlinkMetricCV.java index 0e4d6fcf7b..1a8737a0a0 100644 --- a/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/FlinkMetricCV.java +++ b/streampark-flink/streampark-flink-kubernetes/src/main/java/org/apache/streampark/flink/kubernetes/model/FlinkMetricCV.java @@ -20,7 +20,7 @@ import lombok.AllArgsConstructor; import lombok.Builder; -/** flink cluster metric info */ +/** Immutable snapshot of the aggregate metrics reported by a Flink cluster. */ @Builder @AllArgsConstructor public class FlinkMetricCV { @@ -119,7 +119,7 @@ public Integer totalJob() { public boolean equalsPayload(FlinkMetricCV another) { return java.util.Objects.equals(groupId, another.groupId()) - && totalJmMemory.equals(another.totalTmMemory()) + && totalJmMemory.equals(another.totalJmMemory()) && totalTmMemory.equals(another.totalTmMemory()) && totalTm.equals(another.totalTm()) && totalSlot.equals(another.totalSlot()) diff --git a/streampark-flink/streampark-flink-kubernetes/src/test/java/org/apache/streampark/flink/kubernetes/model/FlinkMetricCVTest.java b/streampark-flink/streampark-flink-kubernetes/src/test/java/org/apache/streampark/flink/kubernetes/model/FlinkMetricCVTest.java new file mode 100644 index 0000000000..0670fd067f --- /dev/null +++ b/streampark-flink/streampark-flink-kubernetes/src/test/java/org/apache/streampark/flink/kubernetes/model/FlinkMetricCVTest.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.kubernetes.model; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class FlinkMetricCVTest { + + @Test + void compareJmAndTmMemory() { + FlinkMetricCV baseline = metric(512, 1024); + + assertThat(baseline.equalsPayload(metric(512, 1024))).isTrue(); + assertThat(baseline.equalsPayload(metric(256, 1024))).isFalse(); + assertThat(baseline.equalsPayload(metric(512, 2048))).isFalse(); + } + + private static FlinkMetricCV metric(int jmMemory, int tmMemory) { + return FlinkMetricCV.builder() + .groupId("default") + .totalJmMemory(jmMemory) + .totalTmMemory(tmMemory) + .pollAckTime(1L) + .build(); + } +} From 551c0a5ebb28bfe4fccf5623df961150056fdde3 Mon Sep 17 00:00:00 2001 From: benjobs Date: Fri, 4 Sep 2026 02:29:55 +0800 Subject: [PATCH 6/9] [Flink] Add Hadoop test dependencies to packer --- streampark-flink/streampark-flink-packer/pom.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/streampark-flink/streampark-flink-packer/pom.xml b/streampark-flink/streampark-flink-packer/pom.xml index b414426bf9..c3eaaf07fb 100644 --- a/streampark-flink/streampark-flink-packer/pom.xml +++ b/streampark-flink/streampark-flink-packer/pom.xml @@ -141,6 +141,19 @@ lombok + + + org.apache.hadoop + hadoop-client-api + test + + + + org.apache.hadoop + hadoop-client-runtime + test + + com.fasterxml.jackson.core From f55749153c69c79526927a9f173194abfd0bf801 Mon Sep 17 00:00:00 2001 From: benjobs Date: Fri, 4 Sep 2026 02:45:40 +0800 Subject: [PATCH 7/9] [Build] Update distribution dependency allowlist --- tools/dependencies/known-dependencies.txt | 76 +++++------------------ 1 file changed, 15 insertions(+), 61 deletions(-) diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 26f663a179..9861d3e7ae 100644 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -11,6 +11,7 @@ aether-transport-file-1.1.0.jar aether-transport-http-1.1.0.jar aether-util-1.1.0.jar aircompressor-0.10.jar +annotations-13.0.jar antlr4-runtime-4.8-1.jar aopalliance-repackaged-2.6.1.jar arrow-format-2.0.0.jar @@ -37,13 +38,11 @@ checker-qual-3.5.0.jar chill-java-0.9.5.jar chill_2.12-0.9.5.jar classmate-1.5.1.jar -commons-beanutils-1.9.3.jar commons-cli-1.5.0.jar commons-codec-1.15.jar commons-collections-3.2.2.jar commons-compiler-3.1.9.jar commons-compress-1.21.jar -commons-configuration2-2.1.1.jar commons-crypto-1.1.0.jar commons-email-1.5.jar commons-io-2.6.jar @@ -56,10 +55,9 @@ commons-text-1.10.0.jar compress-lzf-1.0.3.jar config-1.4.2.jar content-type-2.1.jar -curator-client-2.12.0.jar +curator-client-2.13.0.jar curator-framework-2.13.0.jar curator-recipes-2.13.0.jar -dnsjava-2.1.7.jar docker-java-api-3.3.6.jar docker-java-core-3.3.6.jar docker-java-transport-3.3.6.jar @@ -92,34 +90,20 @@ flink-streaming-java-1.20.1.jar flink-table-api-java-1.20.1.jar flink-table-common-1.20.1.jar freemarker-2.3.30.jar -gson-2.9.1.jar guava-30.0-jre.jar h2-2.1.214.jar -hadoop-annotations-3.2.0.jar -hadoop-auth-3.2.0.jar -hadoop-client-3.2.0.jar -hadoop-client-api-3.3.4.jar -hadoop-client-runtime-3.3.4.jar -hadoop-common-3.2.0.jar -hadoop-hdfs-client-3.2.0.jar -hadoop-mapreduce-client-common-3.2.0.jar -hadoop-mapreduce-client-core-3.2.0.jar -hadoop-mapreduce-client-jobclient-3.2.0.jar -hadoop-yarn-api-3.2.0.jar -hadoop-yarn-client-3.2.0.jar -hadoop-yarn-common-3.2.0.jar +hadoop-client-api-3.3.5.jar +hadoop-client-runtime-3.3.5.jar hibernate-validator-6.2.5.Final.jar hive-storage-api-2.7.2.jar hk2-api-2.6.1.jar hk2-locator-2.6.1.jar hk2-utils-2.6.1.jar -htrace-core4-4.1.0-incubating.jar -httpclient-4.5.14.jar httpclient5-5.1.jar httpclient5-fluent-5.1.jar -httpcore-4.4.16.jar httpcore5-5.1.5.jar httpcore5-h2-5.1.5.jar +icu4j-67.1.jar ivy-2.4.0.jar j2objc-annotations-1.3.jar jackson-annotations-2.13.5.jar @@ -128,27 +112,20 @@ jackson-core-asl-1.9.13.jar jackson-databind-2.13.5.jar jackson-datatype-jdk8-2.13.5.jar jackson-datatype-jsr310-2.13.5.jar -jackson-jaxrs-base-2.13.5.jar -jackson-jaxrs-json-provider-2.13.5.jar jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations-2.13.5.jar jackson-module-parameter-names-2.13.5.jar jackson-module-scala_2.12-2.13.5.jar -jakarta.activation-api-1.2.2.jar jakarta.annotation-api-1.3.5.jar jakarta.inject-2.6.1.jar jakarta.servlet-api-4.0.4.jar jakarta.validation-api-2.0.2.jar jakarta.websocket-api-1.1.2.jar jakarta.ws.rs-api-2.1.6.jar -jakarta.xml.bind-api-2.3.3.jar janino-3.1.9.jar java-jwt-4.0.0.jar javassist-3.24.0-GA.jar -javax.activation-api-1.2.0.jar javax.annotation-api-1.3.2.jar javax.mail-1.5.6.jar -jaxb-api-2.3.1.jar jboss-logging-3.4.3.Final.jar jboss-threads-3.1.0.Final.jar jcip-annotations-1.0-1.jar @@ -160,35 +137,20 @@ jersey-container-servlet-2.35.jar jersey-container-servlet-core-2.35.jar jersey-hk2-2.35.jar jersey-server-2.35.jar -jetty-servlet-9.4.51.v20230217.jar -jetty-util-9.4.51.v20230217.jar -jetty-util-ajax-9.4.51.v20230217.jar jna-5.13.0.jar -jsr305-1.3.9.jar jsch-0.2.11.jar json-smart-2.4.10.jar json4s-ast_2.12-3.7.0-M11.jar json4s-core_2.12-3.7.0-M11.jar json4s-jackson_2.12-3.7.0-M11.jar json4s-scalap_2.12-3.7.0-M11.jar -jsp-api-2.1.jar jsqlparser-4.4.jar +jsr305-1.3.9.jar jul-to-slf4j-1.7.36.jar -kafka-clients-3.1.2.jar -kerb-admin-1.0.1.jar -kerb-client-1.0.1.jar -kerb-common-1.0.1.jar -kerb-core-1.0.1.jar -kerb-crypto-1.0.1.jar -kerb-identity-1.0.1.jar -kerb-server-1.0.1.jar -kerb-simplekdc-1.0.1.jar -kerb-util-1.0.1.jar -kerby-asn1-1.0.1.jar -kerby-config-1.0.1.jar -kerby-pkix-1.0.1.jar -kerby-util-1.0.1.jar -kerby-xdr-1.0.1.jar +kotlin-stdlib-1.6.21.jar +kotlin-stdlib-common-1.6.21.jar +kotlin-stdlib-jdk7-1.6.21.jar +kotlin-stdlib-jdk8-1.6.21.jar kryo-2.24.0.jar kryo-shaded-4.0.2.jar lang-tag-1.4.4.jar @@ -258,7 +220,6 @@ netty-resolver-dns-native-macos-4.1.91.Final-osx-x86_64.jar netty-transport-4.1.91.Final.jar netty-transport-classes-epoll-4.1.91.Final.jar netty-transport-classes-kqueue-4.1.91.Final.jar -netty-transport-native-epoll-4.1.91.Final.jar netty-transport-native-epoll-4.1.91.Final-linux-aarch_64.jar netty-transport-native-epoll-4.1.91.Final-linux-x86_64.jar netty-transport-native-kqueue-4.1.91.Final-osx-aarch_64.jar @@ -270,8 +231,9 @@ netty-transport-udt-4.1.91.Final.jar nimbus-jose-jwt-8.22.1.jar oauth2-oidc-sdk-8.22.jar objenesis-3.2.jar -okhttp-2.7.5.jar -okio-1.6.0.jar +okhttp-4.11.0.jar +okio-3.5.0.jar +okio-jvm-3.5.0.jar orc-core-1.5.12.jar orc-mapreduce-1.5.12.jar orc-shims-1.5.12.jar @@ -302,7 +264,6 @@ postgresql-42.5.1.jar py4j-0.10.9.jar pyrolite-4.30.jar quartz-2.3.2.jar -re2j-1.1.jar scala-compiler-2.12.17.jar scala-library-2.12.17.jar scala-parser-combinators_2.12-1.1.2.jar @@ -336,10 +297,8 @@ spark-network-common_2.12-3.1.2.jar spark-network-shuffle_2.12-3.1.2.jar spark-sketch_2.12-3.1.2.jar spark-sql_2.12-3.1.2.jar -spark-streaming-kafka-0-10_2.12-3.1.2.jar spark-streaming_2.12-3.1.2.jar spark-tags_2.12-3.1.2.jar -spark-token-provider-kafka-0-10_2.12-3.1.2.jar spark-unsafe_2.12-3.1.2.jar spring-aop-5.3.27.jar spring-beans-5.3.27.jar @@ -351,8 +310,8 @@ spring-boot-starter-aop-2.7.11.jar spring-boot-starter-jdbc-2.7.11.jar spring-boot-starter-json-2.7.11.jar spring-boot-starter-quartz-2.7.11.jar -spring-boot-starter-validation-2.7.11.jar spring-boot-starter-undertow-2.7.11.jar +spring-boot-starter-validation-2.7.11.jar spring-boot-starter-web-2.7.11.jar spring-boot-starter-websocket-2.7.11.jar spring-context-5.3.27.jar @@ -367,13 +326,11 @@ spring-tx-5.3.27.jar spring-web-5.3.27.jar spring-webmvc-5.3.27.jar spring-websocket-5.3.27.jar -stax2-api-3.1.4.jar stream-2.9.6.jar streampark-shaded-jackson-1.0.0.jar streampark-shaded-slf4j-1.0.0.jar threeten-extra-1.5.0.jar tika-core-1.20.jar -token-provider-1.0.1.jar tomcat-embed-el-9.0.74.jar undertow-core-2.2.24.Final.jar undertow-servlet-2.2.24.Final.jar @@ -382,12 +339,9 @@ univocity-parsers-2.9.1.jar unused-1.0.0.jar wildfly-client-config-1.0.1.Final.jar wildfly-common-1.5.4.Final.jar -woodstox-core-5.0.3.jar xbean-asm7-shaded-4.15.jar xml-apis-1.4.01.jar xnio-api-3.8.7.Final.jar xnio-nio-3.8.7.Final.jar xz-1.5.jar -zookeeper-3.6.3.jar -zookeeper-jute-3.6.3.jar -icu4j-67.1.jar +zookeeper-3.4.14.jar From 8da50cf20ea1f4a3c5a2cd8ad1daabf3e66623cf Mon Sep 17 00:00:00 2001 From: benjobs Date: Fri, 4 Sep 2026 12:59:46 +0800 Subject: [PATCH 8/9] [Flink] Harden client submission architecture --- AGENTS.md | 4 +- ...ils.java => FlinkConfigurationLoader.java} | 6 +- .../util/FlinkConfigurationUtilsTest.java | 54 ++-- .../console/base/util/BashJavaUtils.java | 4 +- .../service/impl/FlinkClusterServiceImpl.java | 5 +- .../impl/FlinkSavepointServiceImpl.java | 16 +- .../service/impl/FlinkSqlServiceImpl.java | 2 +- .../console/core/util/FlinkEnvUtils.java | 10 +- .../service/FlinkSavepointServiceTest.java | 4 +- .../console/core/util/FlinkEnvUtilsTest.java | 8 +- .../streampark/flink/client/FlinkClient.java | 114 +++++-- .../{proxy => client}/FlinkShimsProxy.java | 78 +++-- ...quest.java => AbstractClusterRequest.java} | 24 +- .../request/AbstractSavepointRequest.java | 95 ++++++ .../flink/client/request/CancelRequest.java | 2 +- .../client/request/ClientRequestUtils.java | 15 +- .../flink/client/request/DeployRequest.java | 2 +- .../client/request/KubernetesDeploySpec.java | 9 +- .../client/request/SavepointRequest.java | 74 ++--- .../flink/client/request/ShutdownRequest.java | 2 +- .../client/request/SubmitApplicationSpec.java | 7 +- .../client/request/SubmitClusterSpec.java | 7 +- .../flink/client/request/SubmitRequest.java | 9 +- .../request/TriggerSavepointRequest.java | 57 ---- .../FlinkShimsProxyTest.java | 2 +- .../request/ClientRequestUtilsTest.java | 11 +- .../streampark-flink-client-core/pom.xml | 7 + .../flink/client/FlinkClientEntrypoint.java | 41 +-- .../client/bean/FlinkJobGraphBuilder.java | 40 ++- .../flink/client/bean/RemoteWorkspace.java | 39 ++- .../client/bean/ResolvedSubmitRequest.java | 180 +++++++++++ .../client/bean/SessionClusterRestClient.java | 135 +++----- .../client/bean/SubmitRequestResolver.java | 294 ++++++++--------- .../FlinkConfigurationBuilder.java} | 296 ++++++++++++------ .../configuration/FlinkConfigurationOps.java | 39 --- .../client/impl/AbstractFlinkClient.java | 157 ++++++---- .../impl/AbstractKubernetesNativeClient.java | 146 ++++----- .../flink/client/impl/AbstractYarnClient.java | 113 ++++--- .../KubernetesNativeApplicationClient.java | 44 +-- .../impl/KubernetesNativeSessionClient.java | 191 +++++------ .../flink/client/impl/LocalClient.java | 44 +-- .../flink/client/impl/RemoteClient.java | 107 ++++--- .../client/impl/YarnApplicationClient.java | 66 ++-- .../flink/client/impl/YarnPerJobClient.java | 33 +- .../flink/client/impl/YarnSessionClient.java | 285 ++++++++--------- .../bean/SessionClusterRestClientTest.java | 99 ++++++ .../FlinkConfigurationBuilderTest.java | 107 +++++++ .../test/SubmitRequestResolverTest.java | 118 +++++-- .../flink/client/test/YarnPerJobTestCase.java | 91 +++--- .../flink/configuration/package-info.java | 4 +- .../flink/core/FlinkTableInitializer.java | 4 +- 51 files changed, 1989 insertions(+), 1312 deletions(-) rename streampark-common/src/main/java/org/apache/streampark/common/util/{FlinkConfigurationUtils.java => FlinkConfigurationLoader.java} (99%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/{proxy => client}/FlinkShimsProxy.java (79%) rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/{ClusterRequest.java => AbstractClusterRequest.java} (72%) create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/AbstractSavepointRequest.java delete mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/TriggerSavepointRequest.java rename streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/{proxy => client}/FlinkShimsProxyTest.java (98%) create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/ResolvedSubmitRequest.java rename streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/{bean/SubmissionConfigurationBuilder.java => configuration/FlinkConfigurationBuilder.java} (56%) delete mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationOps.java create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/bean/SessionClusterRestClientTest.java create mode 100644 streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationBuilderTest.java diff --git a/AGENTS.md b/AGENTS.md index 4f140b584b..b834a69455 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,13 +47,13 @@ The strongest compatibility contracts are the common configuration keys and valu - **Workspace initialization**: `Workspace.LOCAL`, `Workspace.REMOTE`, and derived constants are initialized from one immutable snapshot. Spring bootstrap must publish its completed configuration before workspace paths are first accessed, and must retain the `Workspace.verifyInitializedFrom` guard. -- **Flink YAML compatibility** (`FlinkConfigurationUtils`): Before Flink 1.19, load only `flink-conf.yaml` with the legacy line parser. Flink 1.19 and 1.20 support both names and prefer `flink-conf.yaml` when both exist; parser selection follows the selected filename. Flink 2.0 and later load only `config.yaml` with standard nested YAML parsing. Directory-based loading must always receive the target Flink version. This common utility returns engine-neutral maps; conversion to Flink's `Configuration` belongs in a Flink module. +- **Flink YAML compatibility** (`FlinkConfigurationLoader`): Before Flink 1.19, load only `flink-conf.yaml` with the legacy line parser. Flink 1.19 and 1.20 support both names and prefer `flink-conf.yaml` when both exist; parser selection follows the selected filename. Flink 2.0 and later load only `config.yaml` with standard nested YAML parsing. Directory-based loading must always receive the target Flink version. This common utility returns engine-neutral maps; conversion to Flink's `Configuration` belongs in a Flink module. - **`FlinkShimsProxy`**: The multi-version classloader isolation mechanism uses `ChildFirstClassLoader` and serializes request/response objects across the boundary. Classloaders are cached by concrete Flink version. Never pass target-runtime Flink objects into the parent classloader or introduce target-specific static state that can leak across classloaders. - **Shims modules**: `streampark-flink-shims-base` contains only contracts and implementation shared by every supported shim. Supported concrete modules are Flink 1.18, 1.19, 1.20, 2.0, 2.1, 2.2, and 2.3. APIs removed from Flink 2.x, such as legacy `registerDataStream` and table-function `registerFunction` overloads, belong only in the compatible 1.x implementations. Do not add classes or methods anywhere under `streampark-flink-shims` without an explicit shims architecture decision. -- **Flink submission flow**: `FlinkClient` and its request/response packages form the stable API. `FlinkClientEntrypoint`, `SubmitRequestResolver`, `SubmissionConfigurationBuilder`, and deployment-specific clients own request normalization, configuration assembly, and submission. Keep Flink-version calls behind the shims boundary. +- **Flink submission flow**: `FlinkClient` and its request/response packages form the stable API. `FlinkClientEntrypoint`, `SubmitRequestResolver`, `FlinkConfigurationBuilder`, and deployment-specific clients own request normalization, configuration assembly, and submission. Keep Flink-version calls behind the shims boundary. - **`FlinkApplicationController` / `FlinkApplicationManageService` / `FlinkApplicationActionService`**: The core application management flow. Operations (start, stop, cancel, deploy) must be idempotent and handle all Flink states correctly. The `AppChangeEvent` annotation triggers state synchronization. diff --git a/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java b/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationLoader.java similarity index 99% rename from streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java rename to streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationLoader.java index 1bac3449e9..65a3878c53 100644 --- a/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationUtils.java +++ b/streampark-common/src/main/java/org/apache/streampark/common/util/FlinkConfigurationLoader.java @@ -48,7 +48,7 @@ *

Directory-based loading always requires the target Flink version. Callers that already know * the selected file or transported YAML format can use the corresponding explicit entry point. */ -public final class FlinkConfigurationUtils { +public final class FlinkConfigurationLoader { /** Filename used by Flink's historical flat configuration syntax. */ public static final String LEGACY_FLINK_CONF_FILENAME = "flink-conf.yaml"; @@ -65,9 +65,9 @@ public final class FlinkConfigurationUtils { private static final Logger LOG = StreamParkLoggerFactory.loggerFactory() - .getLogger(FlinkConfigurationUtils.class.getName()); + .getLogger(FlinkConfigurationLoader.class.getName()); - private FlinkConfigurationUtils() { + private FlinkConfigurationLoader() { } /** diff --git a/streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java b/streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java index 21a4a70db9..06902dad27 100644 --- a/streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java +++ b/streampark-common/src/test/java/org/apache/streampark/common/util/FlinkConfigurationUtilsTest.java @@ -38,7 +38,7 @@ class FlinkConfigurationUtilsTest { @Test void loadNestedYamlAndPreserveLists() throws Exception { - Path configFile = temporaryDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME); + Path configFile = temporaryDirectory.resolve(FlinkConfigurationLoader.FLINK_CONF_FILENAME); Files.writeString( configFile, "env:\n" @@ -49,7 +49,7 @@ void loadNestedYamlAndPreserveLists() throws Exception { + " namespaces: [primary, secondary]\n"); Map configuration = - FlinkConfigurationUtils.loadConfigurationFromFile(configFile.toFile()); + FlinkConfigurationLoader.loadConfigurationFromFile(configFile.toFile()); assertThat(configuration.get("env.java.opts.all")) .isEqualTo("--add-opens=java.base/java.util=ALL-UNNAMED"); @@ -60,17 +60,17 @@ void loadNestedYamlAndPreserveLists() throws Exception { @Test void selectOnlyLegacyYamlBeforeFlink119() throws Exception { Files.writeString( - temporaryDirectory.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME), + temporaryDirectory.resolve(FlinkConfigurationLoader.LEGACY_FLINK_CONF_FILENAME), "pipeline.name: legacy\n"); Files.writeString( - temporaryDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME), + temporaryDirectory.resolve(FlinkConfigurationLoader.FLINK_CONF_FILENAME), "pipeline:\n name: standard\n"); Map configuration = - FlinkConfigurationUtils.loadConfiguration( + FlinkConfigurationLoader.loadConfiguration( temporaryDirectory.toString(), "1.18.2"); - assertThat(FlinkConfigurationUtils.usesStandardYaml( + assertThat(FlinkConfigurationLoader.usesStandardYaml( temporaryDirectory.toString(), "1.18.2")).isFalse(); assertThat(configuration.get("pipeline.name")).isEqualTo("legacy"); } @@ -78,26 +78,26 @@ void selectOnlyLegacyYamlBeforeFlink119() throws Exception { @Test void selectFlink119YamlByPrecedence() throws Exception { Path legacyFile = - temporaryDirectory.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME); + temporaryDirectory.resolve(FlinkConfigurationLoader.LEGACY_FLINK_CONF_FILENAME); Files.writeString(legacyFile, "pipeline.name: legacy\n"); Files.writeString( - temporaryDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME), + temporaryDirectory.resolve(FlinkConfigurationLoader.FLINK_CONF_FILENAME), "pipeline:\n name: standard\n"); - assertThat(FlinkConfigurationUtils.usesStandardYaml( + assertThat(FlinkConfigurationLoader.usesStandardYaml( temporaryDirectory.toString(), "1.19.0")).isFalse(); assertThat( - FlinkConfigurationUtils.loadConfiguration( + FlinkConfigurationLoader.loadConfiguration( temporaryDirectory.toString(), "1.19.0") .get("pipeline.name")) .isEqualTo("legacy"); Files.delete(legacyFile); - assertThat(FlinkConfigurationUtils.usesStandardYaml( + assertThat(FlinkConfigurationLoader.usesStandardYaml( temporaryDirectory.toString(), "1.19.1")).isTrue(); assertThat( - FlinkConfigurationUtils.loadConfiguration( + FlinkConfigurationLoader.loadConfiguration( temporaryDirectory.toString(), "1.19.1") .get("pipeline.name")) .isEqualTo("standard"); @@ -106,26 +106,26 @@ void selectFlink119YamlByPrecedence() throws Exception { @Test void supportLegacyThroughFlink120() throws Exception { Path legacyFile = - temporaryDirectory.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME); + temporaryDirectory.resolve(FlinkConfigurationLoader.LEGACY_FLINK_CONF_FILENAME); Files.writeString(legacyFile, "pipeline.name: legacy\n"); Files.writeString( - temporaryDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME), + temporaryDirectory.resolve(FlinkConfigurationLoader.FLINK_CONF_FILENAME), "pipeline:\n name: standard\n"); - assertThat(FlinkConfigurationUtils.usesStandardYaml( + assertThat(FlinkConfigurationLoader.usesStandardYaml( temporaryDirectory.toString(), "1.20.0")).isFalse(); assertThat( - FlinkConfigurationUtils.loadConfiguration( + FlinkConfigurationLoader.loadConfiguration( temporaryDirectory.toString(), "1.20.0") .get("pipeline.name")) .isEqualTo("legacy"); Files.delete(legacyFile); - assertThat(FlinkConfigurationUtils.usesStandardYaml( + assertThat(FlinkConfigurationLoader.usesStandardYaml( temporaryDirectory.toString(), "1.20.3")).isTrue(); assertThat( - FlinkConfigurationUtils.loadConfiguration( + FlinkConfigurationLoader.loadConfiguration( temporaryDirectory.toString(), "1.20.3") .get("pipeline.name")) .isEqualTo("standard"); @@ -134,23 +134,23 @@ void supportLegacyThroughFlink120() throws Exception { @Test void requireConfigYamlFromFlink2() throws Exception { Files.writeString( - temporaryDirectory.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME), + temporaryDirectory.resolve(FlinkConfigurationLoader.LEGACY_FLINK_CONF_FILENAME), "pipeline.name: legacy\n"); assertThatThrownBy( - () -> FlinkConfigurationUtils.loadConfiguration( + () -> FlinkConfigurationLoader.loadConfiguration( temporaryDirectory.toString(), "2.0.0")) .isInstanceOf(ConfigException.class) - .hasMessageContaining(FlinkConfigurationUtils.FLINK_CONF_FILENAME); + .hasMessageContaining(FlinkConfigurationLoader.FLINK_CONF_FILENAME); Files.writeString( - temporaryDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME), + temporaryDirectory.resolve(FlinkConfigurationLoader.FLINK_CONF_FILENAME), "pipeline:\n name: standard\n"); - assertThat(FlinkConfigurationUtils.usesStandardYaml( + assertThat(FlinkConfigurationLoader.usesStandardYaml( temporaryDirectory.toString(), "2.0.0")).isTrue(); assertThat( - FlinkConfigurationUtils.loadConfiguration( + FlinkConfigurationLoader.loadConfiguration( temporaryDirectory.toString(), "2.0.0") .get("pipeline.name")) .isEqualTo("standard"); @@ -159,7 +159,7 @@ void requireConfigYamlFromFlink2() throws Exception { @Test void readLegacyFileAsUtf8() throws Exception { Path file = - temporaryDirectory.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME); + temporaryDirectory.resolve(FlinkConfigurationLoader.LEGACY_FLINK_CONF_FILENAME); Files.writeString( file, "# comment-only lines must be ignored\n" @@ -167,7 +167,7 @@ void readLegacyFileAsUtf8() throws Exception { StandardCharsets.UTF_8); Map configuration = - FlinkConfigurationUtils.loadConfigurationFromFile(file.toFile()); + FlinkConfigurationLoader.loadConfigurationFromFile(file.toFile()); assertThat(configuration.get("pipeline.name")).isEqualTo("\u4e2d\u6587\u4f5c\u4e1a"); } @@ -175,7 +175,7 @@ void readLegacyFileAsUtf8() throws Exception { @Test void rejectMissingConfigurationDirectory() { assertThatThrownBy( - () -> FlinkConfigurationUtils.loadConfiguration( + () -> FlinkConfigurationLoader.loadConfiguration( temporaryDirectory.resolve("missing").toString(), "1.20.0")) .isInstanceOf(ConfigException.class) .hasMessageContaining("does not exist"); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java index b41e0a31d5..2e9577b6ee 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/BashJavaUtils.java @@ -20,7 +20,7 @@ import org.apache.streampark.common.configuration.ConfigurationParser; import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.util.FileUtils; -import org.apache.streampark.common.util.FlinkConfigurationUtils; +import org.apache.streampark.common.util.FlinkConfigurationLoader; import org.apache.commons.io.output.NullOutputStream; @@ -84,7 +84,7 @@ public static void main(String[] args) throws IOException { String version = flinkVersion.majorVersion(); Map config = - FlinkConfigurationUtils.loadConfiguration( + FlinkConfigurationLoader.loadConfiguration( new File(flinkHome, "conf").getAbsolutePath(), version); String flinkPort = config.getOrDefault("rest.port", "8081"); System.setOut(originalOut); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkClusterServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkClusterServiceImpl.java index eeaa3790cd..f88b5c5af4 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkClusterServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkClusterServiceImpl.java @@ -172,7 +172,10 @@ public void start(FlinkCluster cluster) { DeployResponse deployResponse = deployInternal(flinkCluster); ApiAlertException.throwIfNull( deployResponse, - "Deploy cluster failed, unknown reason，please check you params or StreamPark error log"); + "Cluster deployment failed for an unknown reason; check the request and StreamPark logs"); + if (deployResponse.error() != null) { + throw new ApiDetailException(deployResponse.error()); + } if (FlinkDeployMode.isYarnSessionMode(flinkCluster.getFlinkDeployModeEnum())) { String address = String.format( "%s/proxy/%s/", YarnUtils.getRMWebAppURL(true), deployResponse.clusterId()); diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java index 9fc8d14635..88be815937 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSavepointServiceImpl.java @@ -49,7 +49,7 @@ import org.apache.streampark.console.core.watcher.FlinkAppHttpWatcher; import org.apache.streampark.flink.client.FlinkClient; import org.apache.streampark.flink.client.request.JobClientTarget; -import org.apache.streampark.flink.client.request.TriggerSavepointRequest; +import org.apache.streampark.flink.client.request.SavepointRequest; import org.apache.streampark.flink.client.response.SavepointResponse; import org.apache.streampark.flink.util.FlinkUtils; @@ -177,7 +177,7 @@ public void trigger(Long appId, @Nullable String savepointPath, @Nullable Boolea FlinkEnv flinkEnv = flinkEnvService.getById(application.getVersionId()); // infer savepoint - TriggerSavepointRequest request = renderTriggerSavepointRequest(savepointPath, nativeFormat, application, + SavepointRequest request = renderTriggerSavepointRequest(savepointPath, nativeFormat, application, flinkEnv); CompletableFuture savepointFuture = CompletableFuture @@ -488,11 +488,11 @@ private void expire(FlinkSavepoint entity) { } @Nonnull - private TriggerSavepointRequest renderTriggerSavepointRequest( - @Nullable String savepointPath, - Boolean nativeFormat, - FlinkApplication application, - FlinkEnv flinkEnv) { + private SavepointRequest renderTriggerSavepointRequest( + @Nullable String savepointPath, + Boolean nativeFormat, + FlinkApplication application, + FlinkEnv flinkEnv) { String customSavepoint = this.getFinalSavepointDir(savepointPath, application); FlinkCluster cluster = flinkClusterService.getById(application.getFlinkClusterId()); @@ -500,7 +500,7 @@ private TriggerSavepointRequest renderTriggerSavepointRequest( Map properties = this.tryGetRestProps(application, cluster); - return new TriggerSavepointRequest( + return new SavepointRequest( application.getId(), FlinkEnvUtils.version(flinkEnv), application.getDeployModeEnum(), diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java index 0790404acd..c2cff122fc 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/FlinkSqlServiceImpl.java @@ -32,8 +32,8 @@ import org.apache.streampark.console.core.service.FlinkSqlService; import org.apache.streampark.console.core.service.application.FlinkApplicationBackupService; import org.apache.streampark.console.core.util.FlinkEnvUtils; +import org.apache.streampark.flink.client.FlinkShimsProxy; import org.apache.streampark.flink.core.FlinkSqlValidationResult; -import org.apache.streampark.flink.proxy.FlinkShimsProxy; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkEnvUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkEnvUtils.java index ab098ed82d..1582460720 100644 --- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkEnvUtils.java +++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/util/FlinkEnvUtils.java @@ -20,7 +20,7 @@ import org.apache.streampark.common.configuration.ConfigException; import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.FlinkConfigurationUtils; +import org.apache.streampark.common.util.FlinkConfigurationLoader; import org.apache.streampark.console.base.exception.ApiAlertException; import org.apache.streampark.console.base.exception.ApiDetailException; import org.apache.streampark.console.core.entity.FlinkEnv; @@ -84,7 +84,7 @@ public static FlinkVersion version(FlinkEnv environment) { /** Parses the stored YAML using the format selected when it was synchronized. */ public static Map configuration(FlinkEnv environment) { - return FlinkConfigurationUtils.loadConfigurationFromString( + return FlinkConfigurationLoader.loadConfigurationFromString( yaml(environment), isStandardYaml(environment)); } @@ -128,18 +128,18 @@ public static boolean isStandardYaml(FlinkEnv environment) { } // Rows written before the marker was introduced use the selection rule of their registered // Flink installation. Newly synchronized rows are independent of later filesystem changes. - return FlinkConfigurationUtils.usesStandardYaml( + return FlinkConfigurationLoader.usesStandardYaml( Path.of(target.getFlinkHome(), "conf").toString(), target.getVersion()); } private static String readConfiguration(String flinkHome, String flinkVersion) { try { File file = - FlinkConfigurationUtils.resolveConfigurationFile( + FlinkConfigurationLoader.resolveConfigurationFile( Path.of(flinkHome, "conf").toString(), flinkVersion); String yaml = Files.readString(file.toPath(), StandardCharsets.UTF_8); String prefix = - FlinkConfigurationUtils.isStandardYaml(file) ? STANDARD_PREFIX : LEGACY_PREFIX; + FlinkConfigurationLoader.isStandardYaml(file) ? STANDARD_PREFIX : LEGACY_PREFIX; return prefix + DeflaterUtils.zipString(yaml); } catch (ConfigException e) { throw new ApiAlertException(e.getMessage(), e); diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/FlinkSavepointServiceTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/FlinkSavepointServiceTest.java index b81378ba11..7ab6f8bbb6 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/FlinkSavepointServiceTest.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/service/FlinkSavepointServiceTest.java @@ -21,7 +21,7 @@ import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.FlinkConfigurationUtils; +import org.apache.streampark.common.util.FlinkConfigurationLoader; import org.apache.streampark.console.SpringUnitTestBase; import org.apache.streampark.console.core.entity.FlinkApplication; import org.apache.streampark.console.core.entity.FlinkApplicationConfig; @@ -150,7 +150,7 @@ void readSavepointFromDeployConfig(@TempDir Path flinkHome) throws Exception { flinkEnv.setScalaVersion("2.12"); Path confDir = Files.createDirectories(flinkHome.resolve("conf")); Files.writeString( - confDir.resolve(FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME), + confDir.resolve(FlinkConfigurationLoader.LEGACY_FLINK_CONF_FILENAME), SAVEPOINT_DIRECTORY.key() + ": hdfs:///test"); FlinkEnvUtils.sync(flinkEnv); flinkEnvService.save(flinkEnv); diff --git a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkEnvUtilsTest.java b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkEnvUtilsTest.java index 79ee2c76b4..8e740e7074 100644 --- a/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkEnvUtilsTest.java +++ b/streampark-console/streampark-console-service/src/test/java/org/apache/streampark/console/core/util/FlinkEnvUtilsTest.java @@ -17,7 +17,7 @@ package org.apache.streampark.console.core.util; -import org.apache.streampark.common.util.FlinkConfigurationUtils; +import org.apache.streampark.common.util.FlinkConfigurationLoader; import org.apache.streampark.console.core.entity.FlinkEnv; import org.junit.jupiter.api.Test; @@ -39,10 +39,10 @@ void preserveFlink120YamlSelection() throws Exception { Path configurationDirectory = Files.createDirectories(flinkHome.resolve("conf")); Path legacyFile = configurationDirectory.resolve( - FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME); + FlinkConfigurationLoader.LEGACY_FLINK_CONF_FILENAME); Files.writeString(legacyFile, "pipeline.name: legacy\n"); Files.writeString( - configurationDirectory.resolve(FlinkConfigurationUtils.FLINK_CONF_FILENAME), + configurationDirectory.resolve(FlinkConfigurationLoader.FLINK_CONF_FILENAME), "pipeline:\n name: standard\n"); FlinkEnv environment = environment("1.20.0"); @@ -85,7 +85,7 @@ private Path createFlinkHome(String directory, String version, String configFile Path conf = Files.createDirectories(home.resolve("conf")); Files.createFile(lib.resolve("flink-dist-" + version + ".jar")); String yaml = - FlinkConfigurationUtils.LEGACY_FLINK_CONF_FILENAME.equals(configFile) + FlinkConfigurationLoader.LEGACY_FLINK_CONF_FILENAME.equals(configFile) ? "parallelism.default: 1\n" : "parallelism:\n default: 1\n"; Files.writeString(conf.resolve(configFile), yaml); diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java index eb0f9da30a..cb5407401b 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkClient.java @@ -20,55 +20,60 @@ import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.flink.client.request.CancelRequest; import org.apache.streampark.flink.client.request.DeployRequest; +import org.apache.streampark.flink.client.request.SavepointRequest; import org.apache.streampark.flink.client.request.ShutdownRequest; import org.apache.streampark.flink.client.request.SubmitRequest; -import org.apache.streampark.flink.client.request.TriggerSavepointRequest; import org.apache.streampark.flink.client.response.CancelResponse; import org.apache.streampark.flink.client.response.DeployResponse; import org.apache.streampark.flink.client.response.SavepointResponse; import org.apache.streampark.flink.client.response.ShutdownResponse; import org.apache.streampark.flink.client.response.SubmitResponse; -import org.apache.streampark.flink.proxy.FlinkShimsProxy; import java.io.IOException; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.security.Permission; -/** Public facade for version-isolated Flink client operations. */ +/** + * Public facade for version-isolated Flink client operations. + * + *

Each operation selects the classloader for the request's registered Flink installation, + * serializes the request into that loader, and invokes {@code FlinkClientEntrypoint} reflectively. + * Responses are serialized back into the caller's classloader. This boundary prevents Flink + * runtime classes from different supported versions from sharing type identity. + */ public final class FlinkClient { private static final String FLINK_CLIENT_ENTRYPOINT_CLASS = "org.apache.streampark.flink.client.FlinkClientEntrypoint"; + // SecurityManager is process-wide, so installation and restoration must be atomic. + private static final Object EXIT_GUARD_LOCK = new Object(); + private FlinkClient() { } - /** Submits a Flink job using the version and deployment mode in the request. */ + /** + * Submits a Flink job using the version and deployment mode in the request. + * + *

Submission is serialized while the process-wide exit guard is installed. The guard keeps + * application code that invokes {@link System#exit(int)} from terminating the Console JVM. + * + * @param submitRequest complete immutable submission input + * @return identifiers and effective configuration returned by the deployment client + */ public static SubmitResponse submit(SubmitRequest submitRequest) { - SecurityManager previousSecurityManager = System.getSecurityManager(); - boolean exitGuardInstalled = false; - try { - try { - System.setSecurityManager(new ExitSecurityManager()); - exitGuardInstalled = true; - } catch (UnsupportedOperationException ignored) { - // JDK 17+ may reject SecurityManager unless -Djava.security.manager=allow is set. - } - return invokeClient( - submitRequest, - submitRequest.flinkVersion(), - SubmitRequest.class, - "submit", - SubmitResponse.class); - } finally { - if (exitGuardInstalled) { - System.setSecurityManager(previousSecurityManager); - } + synchronized (EXIT_GUARD_LOCK) { + return submitWithExitGuard(submitRequest); } } - /** Cancels a running Flink job. */ + /** + * Cancels a running Flink job, optionally stopping it with a savepoint. + * + * @param stopRequest job identity, deployment context, and cancellation options + * @return cancellation result containing the savepoint path when one was created + */ public static CancelResponse cancel(CancelRequest stopRequest) { return invokeClient( stopRequest, @@ -78,7 +83,12 @@ public static CancelResponse cancel(CancelRequest stopRequest) { CancelResponse.class); } - /** Deploys a Flink session cluster. */ + /** + * Deploys a persistent Flink session cluster. + * + * @param deployRequest target version, deployment mode, and platform configuration + * @return deployed cluster identity and REST address + */ public static DeployResponse deploy(DeployRequest deployRequest) { return invokeClient( deployRequest, @@ -88,7 +98,12 @@ public static DeployResponse deploy(DeployRequest deployRequest) { DeployResponse.class); } - /** Shuts down a Flink session cluster. */ + /** + * Shuts down a persistent Flink session cluster. + * + * @param shutdownRequest target cluster identity and deployment context + * @return identity of the cluster accepted for shutdown + */ public static ShutdownResponse shutdown(ShutdownRequest shutdownRequest) { return invokeClient( shutdownRequest, @@ -98,16 +113,51 @@ public static ShutdownResponse shutdown(ShutdownRequest shutdownRequest) { ShutdownResponse.class); } - /** Triggers a savepoint for a running Flink job. */ - public static SavepointResponse triggerSavepoint(TriggerSavepointRequest savepointRequest) { + /** + * Triggers a savepoint for a running Flink job without cancelling it. + * + * @param savepointRequest job identity, target directory, and format selection + * @return completed savepoint location + */ + public static SavepointResponse triggerSavepoint(SavepointRequest savepointRequest) { return invokeClient( savepointRequest, savepointRequest.flinkVersion(), - TriggerSavepointRequest.class, + SavepointRequest.class, "triggerSavepoint", SavepointResponse.class); } + /** Installs the process-wide exit guard for exactly one serialized submission. */ + private static SubmitResponse submitWithExitGuard(SubmitRequest submitRequest) { + SecurityManager previousSecurityManager = System.getSecurityManager(); + boolean exitGuardInstalled = false; + try { + try { + System.setSecurityManager(new ExitSecurityManager()); + exitGuardInstalled = true; + } catch (UnsupportedOperationException ignored) { + // JDK 17+ may reject SecurityManager unless -Djava.security.manager=allow is set. + } + return invokeClient( + submitRequest, + submitRequest.flinkVersion(), + SubmitRequest.class, + "submit", + SubmitResponse.class); + } finally { + if (exitGuardInstalled) { + System.setSecurityManager(previousSecurityManager); + } + } + } + + /** + * Invokes the client entry point inside the target-version classloader. + * + *

Requests and responses are serialized across the boundary so target-runtime Flink classes + * never leak into the Console classloader. + */ private static R invokeClient( Object request, FlinkVersion flinkVersion, @@ -121,8 +171,7 @@ private static R invokeClient( try { Class entrypointClass = classLoader.loadClass(FLINK_CLIENT_ENTRYPOINT_CLASS); Class requestClass = classLoader.loadClass(requestType.getName()); - Method method = entrypointClass.getDeclaredMethod(methodName, requestClass); - method.setAccessible(true); + Method method = entrypointClass.getMethod(methodName, requestClass); Object shimsRequest = FlinkShimsProxy.getObject(classLoader, request, requestClass); Object result = method.invoke(null, shimsRequest); @@ -140,6 +189,7 @@ private static R invokeClient( }); } + /** Preserves unchecked causes raised inside the target classloader invocation. */ private static RuntimeException unwrapInvocationTarget(InvocationTargetException e) { Throwable cause = e.getCause(); if (cause instanceof RuntimeException) { @@ -148,7 +198,7 @@ private static RuntimeException unwrapInvocationTarget(InvocationTargetException if (cause instanceof Error) { throw (Error) cause; } - return new IllegalStateException("Failed to invoke Flink client method", e); + return new IllegalStateException("Failed to invoke Flink client method", cause); } /** Prevents submitted user code from terminating the console JVM. */ diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkShimsProxy.java similarity index 79% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkShimsProxy.java index 5c8081fdca..f85f2c18bf 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/proxy/FlinkShimsProxy.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/FlinkShimsProxy.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.streampark.flink.proxy; +package org.apache.streampark.flink.client; import org.apache.streampark.common.configuration.Constants; import org.apache.streampark.common.configuration.option.CoreOptions; @@ -35,6 +35,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -93,39 +94,55 @@ public final class FlinkShimsProxy extends LoggerSupport { private FlinkShimsProxy() { } - private static Pattern getFlinkShimsResourcePattern(String majorVersion) { - return Pattern.compile( - "flink-(.*)-" + majorVersion + "(.*).jar", - Pattern.CASE_INSENSITIVE | Pattern.DOTALL); - } - /** - * Get shimsClassLoader to execute for java/scala API (SAM {@link Function}). + * Executes an operation inside the classloader for the requested Flink version. * - * @param flinkVersion flinkVersion - * @param func execute function - * @param return type - * @return result of func + * @param flinkVersion target Flink installation and version + * @param operation operation that must resolve target-version classes + * @param operation result type + * @return result returned by the operation */ - public static T proxy(FlinkVersion flinkVersion, Function func) { + public static T proxy( + FlinkVersion flinkVersion, + Function operation) { ClassLoader shimsClassLoader = getFlinkShimsClassLoader(flinkVersion); - return ClassLoaderUtils.runAsClassLoader(shimsClassLoader, () -> func.apply(shimsClassLoader)); + return ClassLoaderUtils.runAsClassLoader( + shimsClassLoader, () -> operation.apply(shimsClassLoader)); } /** - * Get ClassLoader to verify sql. + * Executes SQL validation with the target Flink table libraries available. * - * @param flinkVersion flinkVersion - * @param func execute function - * @param return type - * @return result of func + * @param flinkVersion target Flink installation and version + * @param operation validation operation + * @param operation result type + * @return result returned by the operation */ - public static T proxyVerifySql(FlinkVersion flinkVersion, Function func) { + public static T proxyVerifySql( + FlinkVersion flinkVersion, + Function operation) { ClassLoader shimsClassLoader = getVerifySqlLibClassLoader(flinkVersion); - return ClassLoaderUtils.runAsClassLoader(shimsClassLoader, () -> func.apply(shimsClassLoader)); + return ClassLoaderUtils.runAsClassLoader( + shimsClassLoader, () -> operation.apply(shimsClassLoader)); } - public static T getObject(ClassLoader loader, Object obj, + /** + * Copies a serializable value into the type space of another classloader. + * + *

Flink runtime objects must not cross the shims boundary. Stable StreamPark request and + * response types use this method to transfer values without sharing their defining classes. + * + * @param loader classloader that must define the copied value + * @param obj serializable source value + * @param type target type loaded in the destination type space + * @param copied value type + * @return deserialized value owned by the destination classloader + * @throws IOException if the value cannot be serialized or read + * @throws ClassNotFoundException if the destination loader cannot resolve a serialized type + */ + public static T getObject( + ClassLoader loader, + Object obj, Class type) throws IOException, ClassNotFoundException { try ( ByteArrayOutputStream arrayOutputStream = new ByteArrayOutputStream(); @@ -141,6 +158,14 @@ public static T getObject(ClassLoader loader, Object obj, } } + /** Builds the pattern used to keep the selected concrete shim child-first. */ + private static Pattern getFlinkShimsResourcePattern(String majorVersion) { + return Pattern.compile( + "flink-(.*)-" + majorVersion + "(.*).jar", + Pattern.CASE_INSENSITIVE | Pattern.DOTALL); + } + + /** Returns the cached SQL-validation classloader for a concrete Flink version. */ private static ClassLoader getVerifySqlLibClassLoader(FlinkVersion flinkVersion) { LOG.logInfo("Add verify sql lib,flink version: " + flinkVersion); return VERIFY_SQL_CLASS_LOADER_CACHE.computeIfAbsent( @@ -160,12 +185,14 @@ private static ClassLoader getVerifySqlLibClassLoader(FlinkVersion flinkVersion) }); } + /** Keeps target Flink jars child-first while excluding the selected shim artifact itself. */ private static boolean loadJarFilter(String jarName, FlinkVersion flinkVersion) { Pattern childFirstPattern = getFlinkShimsResourcePattern(flinkVersion.majorVersion()); return FLINK_JAR_PATTERN.matcher(jarName).matches() && !childFirstPattern.matcher(jarName).matches(); } + /** Adds compatible StreamPark integration artifacts in deterministic filename order. */ private static void addShimsUrls(FlinkVersion flinkVersion, Consumer addShimUrl) { String appHome = System.getProperty(CoreOptions.APP_HOME.key()); if (appHome == null) { @@ -197,9 +224,11 @@ private static void addShimsUrls(FlinkVersion flinkVersion, Consumer addSh LOG.logInfo(includeReason + jarName); } } + matched.sort(Comparator.comparing(File::getName)); matched.forEach(addShimUrl); } + /** Returns the inclusion reason for a compatible integration artifact, or {@code null}. */ static String matchShimIncludeReason( String jarName, String majorVersion, String scalaVersion) { if (jarName.startsWith(FLINK_SHIMS_PREFIX)) { @@ -234,6 +263,7 @@ static String matchShimIncludeReason( return null; } + /** Returns the cached submission classloader for a concrete Flink version. */ private static ClassLoader getFlinkShimsClassLoader(FlinkVersion flinkVersion) { LOG.logInfo("add flink shims urls classloader,flink version: " + flinkVersion); return SHIMS_CLASS_LOADER_CACHE.computeIfAbsent( @@ -273,6 +303,7 @@ private static List getFlinkUrls(FlinkVersion flinkVersion, boolean include return urls; } + /** Creates a child-first loader whose parent owns stable StreamPark boundary types. */ private static ClassLoader createClassLoader( FlinkVersion flinkVersion, List urls) { @@ -283,6 +314,7 @@ private static ClassLoader createClassLoader( jarName -> loadJarFilter(jarName, flinkVersion)); } + /** Collects matching jars from a Flink installation directory in deterministic order. */ private static List getFlinkHomeLib( String flinkHome, String childDir, @@ -295,6 +327,7 @@ private static List getFlinkHomeLib( if (files == null) { return Collections.emptyList(); } + Arrays.sort(files, Comparator.comparing(File::getName)); List urls = new ArrayList<>(); for (File f : files) { if (filterFun.test(f)) { @@ -304,6 +337,7 @@ private static List getFlinkHomeLib( return urls; } + /** Converts a local artifact path to a classloader URL with contextual failure reporting. */ private static URL toUrl(File file) { try { return file.toURI().toURL(); diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClusterRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/AbstractClusterRequest.java similarity index 72% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClusterRequest.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/AbstractClusterRequest.java index 8a5665594b..4a82530941 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClusterRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/AbstractClusterRequest.java @@ -25,8 +25,14 @@ import java.io.Serializable; import java.util.Map; -/** Base request for operations on a Flink session cluster. */ -public abstract class ClusterRequest implements Serializable { +/** + * Serializable base request for operations on a Flink session cluster. + * + *

Property values are validated and copied at construction because requests cross the + * version-isolated client classloader boundary. Accessors return defensive map copies so one + * operation cannot mutate a request observed by another operation. + */ +public abstract class AbstractClusterRequest implements Serializable { private static final long serialVersionUID = 1L; @@ -38,13 +44,13 @@ public abstract class ClusterRequest implements Serializable { @Nullable private final KubernetesDeploySpec kubernetesDeploySpec; - protected ClusterRequest( - FlinkVersion flinkVersion, - FlinkDeployMode deployMode, - @Nullable Map properties, - String clusterId, - long id, - @Nullable KubernetesDeploySpec kubernetesDeploySpec) { + protected AbstractClusterRequest( + FlinkVersion flinkVersion, + FlinkDeployMode deployMode, + @Nullable Map properties, + String clusterId, + long id, + @Nullable KubernetesDeploySpec kubernetesDeploySpec) { this.flinkVersion = flinkVersion; this.deployMode = deployMode; this.properties = ClientRequestUtils.toSerializableMap(properties); diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/AbstractSavepointRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/AbstractSavepointRequest.java new file mode 100644 index 0000000000..fdfee1504b --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/AbstractSavepointRequest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.request; + +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.core.FlinkVersion; +import org.apache.streampark.common.enums.FlinkDeployMode; + +import javax.annotation.Nullable; + +import java.io.Serializable; +import java.util.Map; + +/** + * Serializable base request for job operations that can create a savepoint. + * + *

The request keeps deployment identity, job identity, and configuration together while + * concrete requests define the savepoint format and target path. Property maps are defensively + * copied for transfer through the version-isolated client boundary. + */ +public abstract class AbstractSavepointRequest implements Serializable { + + private static final long serialVersionUID = 1L; + + private final long id; + private final FlinkVersion flinkVersion; + private final FlinkDeployMode deployMode; + private final Map properties; + private final JobClientTarget target; + + protected AbstractSavepointRequest( + long id, + FlinkVersion flinkVersion, + FlinkDeployMode deployMode, + @Nullable Map properties, + JobClientTarget target) { + this.id = id; + this.flinkVersion = flinkVersion; + this.deployMode = deployMode; + this.properties = ClientRequestUtils.toSerializableMap(properties); + this.target = target; + } + + public long id() { + return id; + } + + public FlinkVersion flinkVersion() { + return flinkVersion; + } + + public FlinkDeployMode deployMode() { + return deployMode; + } + + public Map properties() { + return ClientRequestUtils.copyPropertiesMap(properties); + } + + public String clusterId() { + return target.clusterId(); + } + + public String jobId() { + return target.jobId(); + } + + public boolean withSavepoint() { + return true; + } + + public abstract String savepointPath(); + + public abstract boolean nativeFormat(); + + public String kubernetesNamespace() { + String namespace = target.kubernetesNamespace(); + return namespace == null ? Constants.DEFAULT : namespace; + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/CancelRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/CancelRequest.java index d3f143e8b6..e44004d9cb 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/CancelRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/CancelRequest.java @@ -25,7 +25,7 @@ import java.util.Map; /** Request to cancel a Flink job, optionally creating a savepoint first. */ -public final class CancelRequest extends SavepointRequest { +public final class CancelRequest extends AbstractSavepointRequest { private static final long serialVersionUID = 1L; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClientRequestUtils.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClientRequestUtils.java index c7d021376d..cda41de80d 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClientRequestUtils.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ClientRequestUtils.java @@ -29,11 +29,18 @@ final class ClientRequestUtils { private ClientRequestUtils() { } + /** Returns a mutable defensive copy for callers that consume request properties. */ static Map copyPropertiesMap( @Nullable Map properties) { return properties == null ? new LinkedHashMap<>() : new LinkedHashMap<>(properties); } + /** + * Validates and copies properties stored on a request crossing the shims classloader boundary. + * + *

Null values carry no configuration meaning and are omitted. Rejecting non-serializable + * values at construction time produces a local error instead of a delayed proxy failure. + */ static Map toSerializableMap( @Nullable Map properties) { Map result = new LinkedHashMap<>(); @@ -42,10 +49,14 @@ static Map toSerializableMap( } for (Map.Entry entry : properties.entrySet()) { Object value = entry.getValue(); + if (value == null) { + continue; + } if (value instanceof Serializable) { result.put(entry.getKey(), (Serializable) value); - } else if (value != null) { - result.put(entry.getKey(), value.toString()); + } else { + throw new IllegalArgumentException( + "Flink client property is not serializable: " + entry.getKey()); } } return result; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/DeployRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/DeployRequest.java index 20611e64e8..92c36d1fb7 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/DeployRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/DeployRequest.java @@ -25,7 +25,7 @@ import java.util.Map; /** Request to deploy a Flink session cluster. */ -public final class DeployRequest extends ClusterRequest { +public final class DeployRequest extends AbstractClusterRequest { private static final long serialVersionUID = 1L; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/KubernetesDeploySpec.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/KubernetesDeploySpec.java index dcde4ebef2..6f7ee19bdd 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/KubernetesDeploySpec.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/KubernetesDeploySpec.java @@ -23,7 +23,12 @@ import java.io.Serializable; -/** Kubernetes-specific settings for session-cluster deployment. */ +/** + * Kubernetes-specific settings for session-cluster deployment. + * + *

A missing REST exposure type is normalized to {@code ClusterIP} at construction, allowing + * deployment clients to consume this object without repeating a platform default. + */ public final class KubernetesDeploySpec implements Serializable { private static final long serialVersionUID = 1L; @@ -33,7 +38,6 @@ public final class KubernetesDeploySpec implements Serializable { private final String kubeConf; private final String serviceAccount; private final String flinkImage; - @Nullable private final FlinkKubernetesRestExposedType flinkRestExposedType; public KubernetesDeploySpec( @@ -72,7 +76,6 @@ public String flinkImage() { return flinkImage; } - @Nullable public FlinkKubernetesRestExposedType flinkRestExposedType() { return flinkRestExposedType; } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SavepointRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SavepointRequest.java index 67419e419c..25a5e37224 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SavepointRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SavepointRequest.java @@ -17,73 +17,41 @@ package org.apache.streampark.flink.client.request; -import org.apache.streampark.common.configuration.Constants; import org.apache.streampark.common.core.FlinkVersion; import org.apache.streampark.common.enums.FlinkDeployMode; import javax.annotation.Nullable; -import java.io.Serializable; import java.util.Map; -/** Base request for job operations that can create a savepoint. */ -public abstract class SavepointRequest implements Serializable { +/** Request to trigger a savepoint for a running Flink job. */ +public final class SavepointRequest extends AbstractSavepointRequest { private static final long serialVersionUID = 1L; - private final long id; - private final FlinkVersion flinkVersion; - private final FlinkDeployMode deployMode; - private final Map properties; - private final JobClientTarget target; + private final String savepointPath; + private final boolean nativeFormat; - protected SavepointRequest( - long id, - FlinkVersion flinkVersion, - FlinkDeployMode deployMode, - @Nullable Map properties, - JobClientTarget target) { - this.id = id; - this.flinkVersion = flinkVersion; - this.deployMode = deployMode; - this.properties = ClientRequestUtils.toSerializableMap(properties); - this.target = target; + public SavepointRequest( + long id, + FlinkVersion flinkVersion, + FlinkDeployMode deployMode, + @Nullable Map properties, + JobClientTarget target, + @Nullable String savepointPath, + boolean nativeFormat) { + super(id, flinkVersion, deployMode, properties, target); + this.savepointPath = savepointPath; + this.nativeFormat = nativeFormat; } - public long id() { - return id; + @Override + public String savepointPath() { + return savepointPath; } - public FlinkVersion flinkVersion() { - return flinkVersion; - } - - public FlinkDeployMode deployMode() { - return deployMode; - } - - public Map properties() { - return ClientRequestUtils.copyPropertiesMap(properties); - } - - public String clusterId() { - return target.clusterId(); - } - - public String jobId() { - return target.jobId(); - } - - public boolean withSavepoint() { - return true; - } - - public abstract String savepointPath(); - - public abstract boolean nativeFormat(); - - public String kubernetesNamespace() { - String namespace = target.kubernetesNamespace(); - return namespace == null ? Constants.DEFAULT : namespace; + @Override + public boolean nativeFormat() { + return nativeFormat; } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ShutdownRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ShutdownRequest.java index 59aaf57242..71d050a37b 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ShutdownRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/ShutdownRequest.java @@ -25,7 +25,7 @@ import java.util.Map; /** Request to shut down a Flink session cluster. */ -public final class ShutdownRequest extends ClusterRequest { +public final class ShutdownRequest extends AbstractClusterRequest { private static final long serialVersionUID = 1L; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitApplicationSpec.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitApplicationSpec.java index f93877e060..214cae628b 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitApplicationSpec.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitApplicationSpec.java @@ -23,7 +23,12 @@ import java.io.Serializable; -/** Application metadata for Flink submit requests. */ +/** + * Immutable application metadata for a Flink submit request. + * + *

The persisted Flink YAML syntax flag travels with the YAML content so Flink 1.19 submissions + * do not infer a different parser from files that changed after the environment was registered. + */ public final class SubmitApplicationSpec implements Serializable { private static final long serialVersionUID = 1L; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitClusterSpec.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitClusterSpec.java index 0d1d4f2fce..33c1d6e397 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitClusterSpec.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitClusterSpec.java @@ -23,7 +23,12 @@ import java.io.Serializable; -/** Cluster deployment context for Flink submit requests. */ +/** + * Optional cluster deployment context for a Flink submit request. + * + *

The context contains deployment-platform identity only; application metadata remains in + * {@link SubmitApplicationSpec}. + */ public final class SubmitClusterSpec implements Serializable { private static final long serialVersionUID = 1L; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitRequest.java index 7eb959f1c4..ce35d02565 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitRequest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/SubmitRequest.java @@ -30,7 +30,14 @@ import java.io.Serializable; import java.util.Map; -/** Immutable input for submitting a Flink job. */ +/** + * Immutable input for submitting a Flink job through the version-isolated client. + * + *

The request deliberately contains only StreamPark types and serializable values. This keeps + * target-version Flink classes behind the shims classloader boundary. Application and cluster + * specifications retain their separate ownership while convenience accessors preserve the + * established client API. + */ public final class SubmitRequest implements Serializable { private static final long serialVersionUID = 1L; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/TriggerSavepointRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/TriggerSavepointRequest.java deleted file mode 100644 index 8e9644e2e1..0000000000 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/main/java/org/apache/streampark/flink/client/request/TriggerSavepointRequest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.client.request; - -import org.apache.streampark.common.core.FlinkVersion; -import org.apache.streampark.common.enums.FlinkDeployMode; - -import javax.annotation.Nullable; - -import java.util.Map; - -/** Trigger savepoint request. */ -public final class TriggerSavepointRequest extends SavepointRequest { - - private static final long serialVersionUID = 1L; - - private final String savepointPath; - private final boolean nativeFormat; - - public TriggerSavepointRequest( - long id, - FlinkVersion flinkVersion, - FlinkDeployMode deployMode, - @Nullable Map properties, - JobClientTarget target, - @Nullable String savepointPath, - boolean nativeFormat) { - super(id, flinkVersion, deployMode, properties, target); - this.savepointPath = savepointPath; - this.nativeFormat = nativeFormat; - } - - @Override - public String savepointPath() { - return savepointPath; - } - - @Override - public boolean nativeFormat() { - return nativeFormat; - } -} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/FlinkShimsProxyTest.java similarity index 98% rename from streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/FlinkShimsProxyTest.java index e352827cfd..11223b19e4 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/proxy/FlinkShimsProxyTest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/FlinkShimsProxyTest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.streampark.flink.proxy; +package org.apache.streampark.flink.client; import org.apache.streampark.common.configuration.option.CoreOptions; import org.apache.streampark.common.core.FlinkVersion; diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/request/ClientRequestUtilsTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/request/ClientRequestUtilsTest.java index 36bff38ae1..ee82e5e72c 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/request/ClientRequestUtilsTest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-api/src/test/java/org/apache/streampark/flink/client/request/ClientRequestUtilsTest.java @@ -19,24 +19,23 @@ import org.junit.jupiter.api.Test; -import java.io.Serializable; import java.util.HashMap; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; class ClientRequestUtilsTest { @Test - void convertNonSerializableValues() { + void rejectsNonSerializableValues() { Map input = new HashMap<>(); input.put("serializable", "value"); input.put("custom", new Object()); - Map result = ClientRequestUtils.toSerializableMap(input); - - assertThat(result).containsEntry("serializable", "value"); - assertThat(result.get("custom")).isNotNull(); + assertThatThrownBy(() -> ClientRequestUtils.toSerializableMap(input)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("custom"); } @Test diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml b/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml index d0f43da7e5..c64c336c1a 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/pom.xml @@ -88,6 +88,13 @@ guava provided + + + + com.squareup.okhttp3 + okhttp + provided + diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/FlinkClientEntrypoint.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/FlinkClientEntrypoint.java index 5fd22071fe..68fc4715ef 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/FlinkClientEntrypoint.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/FlinkClientEntrypoint.java @@ -28,9 +28,9 @@ import org.apache.streampark.flink.client.impl.YarnSessionClient; import org.apache.streampark.flink.client.request.CancelRequest; import org.apache.streampark.flink.client.request.DeployRequest; +import org.apache.streampark.flink.client.request.SavepointRequest; import org.apache.streampark.flink.client.request.ShutdownRequest; import org.apache.streampark.flink.client.request.SubmitRequest; -import org.apache.streampark.flink.client.request.TriggerSavepointRequest; import org.apache.streampark.flink.client.response.CancelResponse; import org.apache.streampark.flink.client.response.DeployResponse; import org.apache.streampark.flink.client.response.SavepointResponse; @@ -64,24 +64,7 @@ public final class FlinkClientEntrypoint { private FlinkClientEntrypoint() { } - @FunctionalInterface - private interface ClientInvoker { - - R invoke(AbstractFlinkClient client) throws FlinkException; - } - - private static R invokeClient( - FlinkDeployMode deployMode, - ClientInvoker invoker, - String action) throws FlinkException { - AbstractFlinkClient client = CLIENTS.get(deployMode); - if (client != null) { - return invoker.invoke(client); - } - throw new UnsupportedOperationException("Unsupported " + deployMode + " " + action); - } - - /** Submits an application through the client selected by its deploy mode. */ + /** Submits a job through the client selected by its deploy mode. */ public static SubmitResponse submit(SubmitRequest submitRequest) throws FlinkException { return invokeClient( submitRequest.deployMode(), client -> client.submit(submitRequest), "submit"); @@ -94,7 +77,7 @@ public static CancelResponse cancel(CancelRequest cancelRequest) throws FlinkExc } /** Triggers a savepoint through the client selected by its deploy mode. */ - public static SavepointResponse triggerSavepoint(TriggerSavepointRequest savepointRequest) throws FlinkException { + public static SavepointResponse triggerSavepoint(SavepointRequest savepointRequest) throws FlinkException { return invokeClient( savepointRequest.deployMode(), client -> client.triggerSavepoint(savepointRequest), @@ -124,4 +107,22 @@ public static ShutdownResponse shutdown(ShutdownRequest shutdownRequest) throws throw new UnsupportedOperationException( "Unsupported " + shutdownRequest.deployMode() + " shutdown cluster "); } + + /** Resolves the deployment client and applies a job-level operation to it. */ + private static R invokeClient( + FlinkDeployMode deployMode, + ClientInvoker invoker, + String action) throws FlinkException { + AbstractFlinkClient client = CLIENTS.get(deployMode); + if (client != null) { + return invoker.invoke(client); + } + throw new UnsupportedOperationException("Unsupported " + deployMode + " " + action); + } + + @FunctionalInterface + private interface ClientInvoker { + + R invoke(AbstractFlinkClient client) throws FlinkException; + } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/FlinkJobGraphBuilder.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/FlinkJobGraphBuilder.java index 794a240375..e5f206b881 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/FlinkJobGraphBuilder.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/FlinkJobGraphBuilder.java @@ -24,6 +24,7 @@ import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.client.program.PackagedProgramUtils; import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.CoreOptions; import org.apache.flink.runtime.jobgraph.JobGraph; import java.io.File; @@ -32,7 +33,13 @@ import java.util.Collections; import java.util.List; -/** Builds a JobGraph with the classloader of the registered Flink version. */ +/** + * Builds a {@link JobGraph} with the classloader of the registered Flink version. + * + *

The builder temporarily installs the target-version parent classloader, constructs the + * packaged program, and restores the original thread context classloader in every outcome. + * Partially created resources are closed without replacing the original construction failure. + */ public final class FlinkJobGraphBuilder { private FlinkJobGraphBuilder() { @@ -41,18 +48,18 @@ private FlinkJobGraphBuilder() { /** Creates the packaged program and its executable job graph. */ public static Result build( Configuration configuration, - SubmitRequest request, + ResolvedSubmitRequest resolved, File jarFile) throws Exception { + SubmitRequest request = resolved.request(); PackagedProgram.Builder builder = PackagedProgram.newBuilder() - .setSavepointRestoreSettings( - SubmitRequestResolver.savepointRestoreSettings(request)) + .setSavepointRestoreSettings(resolved.savepointRestoreSettings()) .setEntryPointClassName( configuration .getOptional(ApplicationConfiguration.APPLICATION_MAIN_CLASS) .orElseThrow( () -> new IllegalStateException( - "Application main class is not configured"))) + "Job main class is not configured"))) .setArguments( configuration .getOptional(ApplicationConfiguration.APPLICATION_ARGS) @@ -62,7 +69,7 @@ public static Result build( if (request.jobType() != FlinkJobType.PYFLINK) { builder.setJarFile(jarFile); boolean flinkSqlJob = request.jobType() == FlinkJobType.FLINK_SQL; - builder.setUserClassPaths(userClassPaths(request, !flinkSqlJob)); + builder.setUserClassPaths(userClassPaths(resolved, !flinkSqlJob)); builder.setConfiguration(parentFirstConfiguration()); } @@ -85,12 +92,14 @@ public static Result build( PackagedProgramUtils.createJobGraph( program, jobGraphConfiguration, - SubmissionConfigurationBuilder.parallelism(request), + configuration.get( + CoreOptions.DEFAULT_PARALLELISM, + CoreOptions.DEFAULT_PARALLELISM.defaultValue()), null, false); return new Result(program, jobGraph); } catch (Exception failure) { - closeAfterBuildFailure(request, program, failure); + closeAfterBuildFailure(program, failure); throw failure; } } finally { @@ -98,13 +107,9 @@ public static Result build( } } + /** Closes a partially built program without hiding the original construction failure. */ private static void closeAfterBuildFailure( - SubmitRequest request, - PackagedProgram program, - Exception failure) { - if (!SubmitRequestResolver.canSafelyClosePackagedProgram(request)) { - return; - } + PackagedProgram program, Exception failure) { try { program.close(); } catch (Exception closeFailure) { @@ -112,6 +117,7 @@ private static void closeAfterBuildFailure( } } + /** Declares packages that must retain one identity across the user-code classloader boundary. */ private static Configuration parentFirstConfiguration() { Configuration configuration = new Configuration(); configuration.setString( @@ -159,8 +165,10 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE *

Thin uploaded jars need {@code flink-dist} on Flink 1.20 and later. SQL fat jars omit it * and use parent-first delegation to prevent dependency conflicts. */ - private static List userClassPaths(SubmitRequest request, boolean includeFlinkDist) { - List classPaths = new ArrayList<>(SubmitRequestResolver.classPaths(request)); + private static List userClassPaths( + ResolvedSubmitRequest resolved, + boolean includeFlinkDist) { + List classPaths = new ArrayList<>(resolved.getSubmissionClassPath()); classPaths.removeIf( url -> { String path = url.getPath(); diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/RemoteWorkspace.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/RemoteWorkspace.java index 38c013f76a..21c54be045 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/RemoteWorkspace.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/RemoteWorkspace.java @@ -27,14 +27,38 @@ import java.io.IOException; import java.io.UncheckedIOException; -/** Resolved HDFS paths used while submitting a Flink job through YARN. */ -public class RemoteWorkspace { +/** + * Immutable remote workspace paths used by YARN submission. + * + *

The local Flink installation name identifies its corresponding directory in the shared + * workspace. Resolving symbolic links keeps this mapping stable when {@code FLINK_HOME} points to + * a version alias. + */ +public final class RemoteWorkspace { private final String flinkDistJar; private final String flinkLib; private final String flinkPlugins; private final String appJars; + private RemoteWorkspace( + String flinkDistJar, + String flinkLib, + String flinkPlugins, + String appJars) { + this.flinkDistJar = flinkDistJar; + this.flinkLib = flinkLib; + this.flinkPlugins = flinkPlugins; + this.appJars = appJars; + } + + /** + * Resolves local and remote artifact paths for a registered Flink installation. + * + * @param flinkVersion registered Flink installation + * @return immutable paths required by YARN submission + * @throws UncheckedIOException if a symbolic Flink home cannot be resolved + */ public static RemoteWorkspace resolve(FlinkVersion flinkVersion) { Workspace workspace = Workspace.REMOTE; String flinkHome = flinkVersion.flinkHome; @@ -57,17 +81,6 @@ public static RemoteWorkspace resolve(FlinkVersion flinkVersion) { workspace.jars); } - private RemoteWorkspace( - String flinkDistJar, - String flinkLib, - String flinkPlugins, - String appJars) { - this.flinkDistJar = flinkDistJar; - this.flinkLib = flinkLib; - this.flinkPlugins = flinkPlugins; - this.appJars = appJars; - } - public String flinkDistJar() { return flinkDistJar; } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/ResolvedSubmitRequest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/ResolvedSubmitRequest.java new file mode 100644 index 0000000000..fef8e57654 --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/ResolvedSubmitRequest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.bean; + +import org.apache.streampark.common.configuration.Constants; +import org.apache.streampark.common.configuration.FlinkOptions; +import org.apache.streampark.common.configuration.option.ApplicationOptions; +import org.apache.streampark.common.enums.FlinkJobType; +import org.apache.streampark.flink.client.configuration.FlinkSavepointOptions; +import org.apache.streampark.flink.client.request.SubmitRequest; + +import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; + +import javax.annotation.Nullable; + +import java.io.File; +import java.net.URL; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Immutable submission values derived from one {@link SubmitRequest} snapshot. + * + *

All filesystem access and build validation happen in {@link SubmitRequestResolver}. This + * object contains only immutable values, so every submission stage observes the same artifacts and + * classpath. + */ +public final class ResolvedSubmitRequest { + + private final SubmitRequest request; + private final Map jobProperties; + private final Map jobOptions; + @Nullable + private final String jobMainClass; + private final String jobName; + @Nullable + private final String flinkSqlContent; + private final boolean nonRestoredStateAllowed; + private final SavepointRestoreSettings savepointRestoreSettings; + @Nullable + private final File userJarFile; + private final List submissionClassPath; + + ResolvedSubmitRequest( + SubmitRequest request, + Map jobConfig, + String jobName, + @Nullable File userJarFile, + List submissionClassPath) { + this.request = Objects.requireNonNull(request, "request must not be null"); + Map config = Map.copyOf(jobConfig); + this.jobProperties = Collections.unmodifiableMap( + filterByPrefix(config, FlinkOptions.PROPERTY_PREFIX.value())); + + this.jobOptions = Collections.unmodifiableMap( + filterByPrefix(config, FlinkOptions.OPTION_PREFIX.value())); + + this.jobMainClass = resolveJobMainClass(config); + + this.jobName = Objects.requireNonNull(jobName, "jobName must not be null"); + + Object sql = request.extraParameter().get(ApplicationOptions.SQL.key()); + this.flinkSqlContent = sql == null ? null : sql.toString(); + + Object allowNonRestored = + request.properties().get(FlinkSavepointOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE.key()); + + this.nonRestoredStateAllowed = + allowNonRestored != null && Boolean.parseBoolean(allowNonRestored.toString()); + + this.savepointRestoreSettings = + request.savePoint() == null || request.savePoint().isEmpty() + ? SavepointRestoreSettings.none() + : SavepointRestoreSettings.forPath( + request.savePoint(), nonRestoredStateAllowed); + this.userJarFile = userJarFile; + this.submissionClassPath = List.copyOf(submissionClassPath); + } + + /** Returns the original immutable submission request. */ + public SubmitRequest request() { + return request; + } + + /** Returns job properties without their transport prefix. */ + public Map jobProperties() { + return jobProperties; + } + + /** Returns job options without their transport prefix. */ + public Map jobOptions() { + return jobOptions; + } + + /** Returns the entry class selected for the submitted job type. */ + @Nullable + public String getJobMainClass() { + return jobMainClass; + } + + /** Returns the validated name used throughout this submission. */ + public String getJobName() { + return jobName; + } + + /** Returns the SQL payload carried by a Flink SQL request. */ + @Nullable + public String getFlinkSqlContent() { + return flinkSqlContent; + } + + /** Returns whether state not represented in a savepoint may be skipped. */ + public boolean isNonRestoredStateAllowed() { + return nonRestoredStateAllowed; + } + + /** Returns restore settings derived from the immutable request snapshot. */ + public SavepointRestoreSettings savepointRestoreSettings() { + return savepointRestoreSettings; + } + + /** Returns the classpath captured for JobGraph construction. */ + public List getSubmissionClassPath() { + return submissionClassPath; + } + + /** Returns the built user JAR required by the selected deployment mode. */ + @Nullable + public File getUserJarFile() { + return userJarFile; + } + + /** Resolves the job entry class without reparsing job configuration. */ + @Nullable + private String resolveJobMainClass(Map jobConfig) { + if (request.jobType() == FlinkJobType.FLINK_SQL) { + return Constants.STREAMPARK_FLINKSQL_CLIENT_CLASS; + } + if (request.jobType() == FlinkJobType.PYFLINK) { + return Constants.PYTHON_FLINK_DRIVER_CLASS_NAME; + } + String mainClass = + jobProperties.get(FlinkOptions.APPLICATION_MAIN_CLASS.key()); + return mainClass == null + ? jobConfig.get(FlinkOptions.APPLICATION_MAIN_CLASS.key()) + : mainClass; + } + + /** Selects non-empty entries under a transport namespace and strips that namespace. */ + private static Map filterByPrefix( + Map values, + String prefix) { + Map result = new HashMap<>(); + for (Map.Entry entry : values.entrySet()) { + String value = entry.getValue(); + if (entry.getKey().startsWith(prefix) && value != null && !value.isEmpty()) { + result.put(entry.getKey().substring(prefix.length()), value); + } + } + return result; + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SessionClusterRestClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SessionClusterRestClient.java index 6cf09638a9..0e314d01b0 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SessionClusterRestClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SessionClusterRestClient.java @@ -19,6 +19,7 @@ import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.JsonUtils; +import org.apache.streampark.common.util.OkHttpUtils; import org.apache.streampark.flink.client.configuration.FlinkSavepointOptions; import org.apache.streampark.shaded.com.fasterxml.jackson.databind.JsonNode; @@ -27,30 +28,24 @@ import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.CoreOptions; -import java.io.ByteArrayInputStream; +import okhttp3.MediaType; +import okhttp3.MultipartBody; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.SequenceInputStream; -import java.io.UncheckedIOException; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.nio.charset.StandardCharsets; import java.security.AccessController; import java.security.PrivilegedExceptionAction; -import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; import java.util.List; /** Submits Flink jobs to session clusters through the Flink REST API. */ public final class SessionClusterRestClient { - private static final Duration CONNECT_TIMEOUT = Duration.ofSeconds(30); - private static final Duration REQUEST_TIMEOUT = Duration.ofMinutes(5); + private static final MediaType JAR_MEDIA_TYPE = + MediaType.parse("application/java-archive"); + private static final MediaType JSON_MEDIA_TYPE = + MediaType.parse("application/json; charset=utf-8"); private SessionClusterRestClient() { } @@ -60,86 +55,55 @@ public static String submit( String jobManagerUrl, File jobJar, Configuration configuration) throws Exception { - AssertUtils.required(jobJar.isFile(), "Flink job JAR does not exist: " + jobJar); - HttpClient httpClient = - HttpClient.newBuilder().connectTimeout(CONNECT_TIMEOUT).build(); - - String boundary = "----StreamParkBoundary" + System.currentTimeMillis(); - HttpRequest uploadRequest = - HttpRequest.newBuilder() - .uri(endpoint(jobManagerUrl, "/jars/upload")) - .timeout(REQUEST_TIMEOUT) - .header("Content-Type", "multipart/form-data; boundary=" + boundary) - .POST(multipartBody(boundary, jobJar)) - .build(); - HttpResponse uploadResponse = send(httpClient, uploadRequest); - requireSuccess("upload job JAR", uploadResponse); - JarUploadResponse upload = parseUploadResponse(uploadResponse.body()); - - HttpRequest runRequest = - HttpRequest.newBuilder() - .uri(endpoint(jobManagerUrl, "/jars/" + upload.jarId() + "/run")) - .timeout(REQUEST_TIMEOUT) - .header("Content-Type", "application/json") - .POST( - HttpRequest.BodyPublishers.ofString( - JsonUtils.write(new JarRunRequest(configuration)))) - .build(); - HttpResponse runResponse = send(httpClient, runRequest); - requireSuccess("start uploaded job", runResponse); - return parseJobId(runResponse.body()); + AssertUtils.required( + jobManagerUrl != null && !jobManagerUrl.isBlank(), + "Flink JobManager URL must not be empty"); + AssertUtils.required( + jobJar != null && jobJar.isFile(), "Flink job JAR does not exist: " + jobJar); + RequestBody uploadBody = new MultipartBody.Builder() + .setType(MultipartBody.FORM) + .addFormDataPart( + "jarfile", jobJar.getName(), RequestBody.create(jobJar, JAR_MEDIA_TYPE)) + .build(); + Request uploadRequest = new Request.Builder() + .url(endpoint(jobManagerUrl, "/jars/upload")) + .post(uploadBody) + .build(); + JarUploadResponse upload = parseUploadResponse(execute("upload job JAR", uploadRequest)); + + RequestBody runBody = RequestBody.create( + JsonUtils.write(new JarRunRequest(configuration)), JSON_MEDIA_TYPE); + Request runRequest = new Request.Builder() + .url(endpoint(jobManagerUrl, "/jars/" + upload.jarId() + "/run")) + .post(runBody) + .build(); + return parseJobId(execute("start uploaded job", runRequest)); } - private static URI endpoint(String jobManagerUrl, String path) { + /** Joins a normalized JobManager base URL with a Flink REST resource path. */ + private static String endpoint(String jobManagerUrl, String path) { String normalized = jobManagerUrl.endsWith("/") ? jobManagerUrl.substring(0, jobManagerUrl.length() - 1) : jobManagerUrl; - return URI.create(normalized + path); - } - - private static HttpResponse send(HttpClient client, HttpRequest request) throws Exception { - return AccessController.doPrivileged( - (PrivilegedExceptionAction>) () -> client.send( - request, - HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8))); + return normalized + path; } - private static void requireSuccess(String action, HttpResponse response) { - int status = response.statusCode(); - if (status < 200 || status >= 300) { - throw new IllegalStateException( - "Failed to " + action + ": HTTP " + status + ", response=" + response.body()); - } - } - - private static HttpRequest.BodyPublisher multipartBody(String boundary, File jobJar) { - byte[] header = - ("--" - + boundary - + "\r\nContent-Disposition: form-data; name=\"jarfile\"; filename=\"" - + jobJar.getName() - + "\"\r\nContent-Type: application/java-archive\r\n\r\n") - .getBytes(StandardCharsets.UTF_8); - byte[] footer = - ("\r\n--" + boundary + "--\r\n").getBytes(StandardCharsets.UTF_8); - return HttpRequest.BodyPublishers.ofInputStream( - () -> multipartStream(header, jobJar, footer)); - } - - private static InputStream multipartStream(byte[] header, File jobJar, byte[] footer) { - try { - List streams = - Arrays.asList( - new ByteArrayInputStream(header), - new FileInputStream(jobJar), - new ByteArrayInputStream(footer)); - return new SequenceInputStream(Collections.enumeration(streams)); - } catch (IOException e) { - throw new UncheckedIOException("Failed to open Flink job JAR: " + jobJar, e); + /** Executes one REST request, consumes its body, and always closes the response. */ + private static String execute(String action, Request request) throws Exception { + try ( + Response response = AccessController.doPrivileged( + (PrivilegedExceptionAction) () -> OkHttpUtils.call(request))) { + String body = response.body() == null ? "" : response.body().string(); + if (!response.isSuccessful()) { + throw new IllegalStateException( + "Failed to " + action + ": HTTP " + response.code() + ", response=" + body); + } + return body; } } + /** Validates the Flink upload response and extracts the server-side JAR identity. */ private static JarUploadResponse parseUploadResponse(String responseBody) { try { JsonNode node = JsonUtils.read(responseBody, JsonNode.class); @@ -159,6 +123,7 @@ private static JarUploadResponse parseUploadResponse(String responseBody) { } } + /** Extracts a job ID while accepting the field casing used by supported Flink versions. */ private static String parseJobId(String responseBody) { try { JsonNode node = JsonUtils.read(responseBody, JsonNode.class); diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequestResolver.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequestResolver.java index c85d15e3b3..d4c532e135 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequestResolver.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmitRequestResolver.java @@ -19,16 +19,12 @@ import org.apache.streampark.common.configuration.ConfigurationFormat; import org.apache.streampark.common.configuration.ConfigurationParser; -import org.apache.streampark.common.configuration.Constants; import org.apache.streampark.common.configuration.FlinkOptions; import org.apache.streampark.common.configuration.Workspace; -import org.apache.streampark.common.configuration.option.ApplicationOptions; import org.apache.streampark.common.enums.FlinkDeployMode; -import org.apache.streampark.common.enums.FlinkJobType; import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.DeflaterUtils; import org.apache.streampark.common.util.HdfsUtils; -import org.apache.streampark.flink.client.configuration.FlinkSavepointOptions; import org.apache.streampark.flink.client.request.SubmitRequest; import org.apache.streampark.flink.packer.pipeline.BuildResult; import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse; @@ -36,21 +32,25 @@ import org.apache.streampark.shaded.com.fasterxml.jackson.core.type.TypeReference; import org.apache.streampark.shaded.com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; - -import javax.annotation.Nullable; - import java.io.File; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; +import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; -/** Resolves runtime submission values from the serialized client request. */ +/** + * Parses transport-oriented submission fields into an immutable request snapshot. + * + *

Job configuration is decoded exactly once. The returned snapshot owns all derived + * values used by configuration assembly and deployment clients, preventing repeated HDFS reads or + * inconsistent results within one submission. + */ public final class SubmitRequestResolver { private static final int CONFIG_SCHEME_LENGTH = 7; @@ -58,182 +58,145 @@ public final class SubmitRequestResolver { private SubmitRequestResolver() { } - /** Returns application properties without their transport prefix. */ - public static Map applicationProperties(SubmitRequest request) { - return getParameters(request, FlinkOptions.PROPERTY_PREFIX.value()); + /** + * Resolves a request from one job-configuration read. + * + * @param request serialized submission request + * @return immutable values derived from the request + */ + public static ResolvedSubmitRequest resolve(SubmitRequest request) { + Objects.requireNonNull(request, "request must not be null"); + Map jobConfig = + request.appConf() == null + ? Collections.emptyMap() + : loadJobConfig(request.appConf()); + String jobName = resolveJobName(request, jobConfig); + AssertUtils.hasText(jobName, "Flink job name must not be blank"); + validateBuildResult(request, jobName); + return new ResolvedSubmitRequest( + request, + jobConfig, + jobName, + resolveUserJar(request), + resolveClassPath(request)); + } + + /** Resolves the explicit job name or the configured pipeline-name fallback. */ + private static String resolveJobName(SubmitRequest request, Map jobConfig) { + if (request.appName() != null) { + return request.appName(); + } + String propertyKey = FlinkOptions.PROPERTY_PREFIX.value() + FlinkOptions.PIPELINE_NAME.key(); + return jobConfig.get(propertyKey); + } + + /** Verifies build state once before any configuration or deployment work begins. */ + private static void validateBuildResult(SubmitRequest request, String jobName) { + BuildResult buildResult = request.buildResult(); + String target = + FlinkDeployMode.isKubernetesMode(request.deployMode()) + ? ", clusterId=" + + request.clusterId() + + ", namespace=" + + request.kubernetesNamespace() + : ""; + AssertUtils.required( + buildResult != null, + "[flink-submit] current job " + + jobName + + " was not yet built; build result is empty" + + target); + AssertUtils.required( + buildResult.pass(), + "[flink-submit] current job " + jobName + " build failed" + target); } - /** Returns application options without their transport prefix. */ - public static Map applicationOptions(SubmitRequest request) { - return getParameters(request, FlinkOptions.OPTION_PREFIX.value()); + /** Captures the built user JAR for modes that upload a local artifact. */ + private static File resolveUserJar(SubmitRequest request) { + if (request.deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) { + return null; + } + ShadedBuildResponse buildResult = request.buildResult().as(ShadedBuildResponse.class); + String shadedJarPath = buildResult.shadedJarPath(); + AssertUtils.hasText(shadedJarPath, "Built user JAR path must not be blank"); + return new File(shadedJarPath); } - /** Resolves the application entry class for the submitted job type. */ - @Nullable - public static String applicationMain(SubmitRequest request) { - if (request.jobType() == FlinkJobType.FLINK_SQL) { - return Constants.STREAMPARK_FLINKSQL_CLIENT_CLASS; - } - if (request.jobType() == FlinkJobType.PYFLINK) { - return Constants.PYTHON_FLINK_DRIVER_CLASS_NAME; + /** Captures the deterministic user-code classpath only for JobGraph deployment modes. */ + private static List resolveClassPath(SubmitRequest request) { + if (!usesJobGraph(request.deployMode())) { + return Collections.emptyList(); } - String mainClass = - applicationProperties(request).get(FlinkOptions.APPLICATION_MAIN_CLASS.key()); - if (mainClass == null && request.appConf() != null) { - mainClass = - loadApplicationConfig(request.appConf()) - .get(FlinkOptions.APPLICATION_MAIN_CLASS.key()); + try { + List classPath = new ArrayList<>(request.flinkVersion().getFlinkLibs()); + classPath.sort(Comparator.comparing(URL::toExternalForm)); + classPath.addAll(listJobLibraries(request)); + return Collections.unmodifiableList(classPath); + } catch (Exception e) { + throw new IllegalStateException("Failed to resolve the Flink submission classpath", e); } - return mainClass; } - /** Resolves the explicit or configured application name. */ - @Nullable - public static String effectiveApplicationName(SubmitRequest request) { - return request.appName() == null - ? applicationProperties(request).get(FlinkOptions.PIPELINE_NAME.key()) - : request.appName(); + /** Returns whether the deployment builds and submits a JobGraph in this client. */ + private static boolean usesJobGraph(FlinkDeployMode deployMode) { + return deployMode == FlinkDeployMode.LOCAL + || deployMode == FlinkDeployMode.REMOTE + || deployMode == FlinkDeployMode.YARN_SESSION + || deployMode == FlinkDeployMode.YARN_PER_JOB; } - /** Returns the application-specific libraries available in the local workspace. */ - public static List libraries(SubmitRequest request) { + /** Lists job-specific libraries from the local workspace in deterministic order. */ + private static List listJobLibraries(SubmitRequest request) { File libDir = - new File( - new File(Workspace.LOCAL.workspace, String.valueOf(request.id())), "lib"); + new File(new File(Workspace.LOCAL.workspace, String.valueOf(request.id())), "lib"); File[] files = libDir.listFiles(); if (files == null) { return Collections.emptyList(); } + Arrays.sort(files, Comparator.comparing(File::getName)); List urls = new ArrayList<>(files.length); for (File file : files) { try { urls.add(file.toURI().toURL()); } catch (MalformedURLException e) { - throw new IllegalArgumentException("Invalid library path: " + file, e); + throw new IllegalArgumentException("Invalid job library path: " + file, e); } } return urls; } - /** Builds the complete user-code classpath for submission. */ - public static List classPaths(SubmitRequest request) { - try { - List classPaths = new ArrayList<>(request.flinkVersion().getFlinkLibs()); - classPaths.addAll(libraries(request)); - return classPaths; - } catch (Exception e) { - throw new IllegalStateException("Failed to resolve the Flink submission classpath", e); - } - } - - /** Returns the SQL payload carried by a Flink SQL request. */ - @Nullable - public static String flinkSql(SubmitRequest request) { - Map extraParameter = request.extraParameter(); - Object sql = extraParameter.get(ApplicationOptions.SQL.key()); - return sql == null ? null : sql.toString(); - } - - /** Returns whether state not represented in a savepoint may be skipped. */ - public static boolean allowNonRestoredState(SubmitRequest request) { - Object value = - request.properties().get(FlinkSavepointOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE.key()); - return value != null && Boolean.parseBoolean(value.toString()); - } - - /** Creates Flink restore settings from the request savepoint options. */ - public static SavepointRestoreSettings savepointRestoreSettings(SubmitRequest request) { - if (request.savePoint() == null || request.savePoint().isEmpty()) { - return SavepointRestoreSettings.none(); - } - return SavepointRestoreSettings.forPath( - request.savePoint(), allowNonRestoredState(request)); - } - - /** Resolves the built user JAR for deployment modes that upload a local artifact. */ - @Nullable - public static File userJarFile(SubmitRequest request) { - if (request.deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) { - return null; - } - validateBuildResult(request); - ShadedBuildResponse buildResult = request.buildResult().as(ShadedBuildResponse.class); - String shadedJarPath = buildResult.shadedJarPath(); - return shadedJarPath == null ? null : new File(shadedJarPath); - } - - /** Returns whether the target Flink version safely supports closing packaged programs. */ - public static boolean canSafelyClosePackagedProgram(SubmitRequest request) { - String[] parts = request.flinkVersion().version().split("\\."); - if (parts.length < 3) { - return false; - } - try { - int major = Integer.parseInt(parts[0].trim()); - int minor = Integer.parseInt(parts[1].trim()); - int patch = Integer.parseInt(parts[2].trim()); - return major >= 1 && (minor > 12 || (minor == 12 && patch >= 2)); - } catch (NumberFormatException ignored) { - return false; - } - } - - /** Verifies that the application artifact was built successfully before submission. */ - public static void validateBuildResult(SubmitRequest request) { - BuildResult buildResult = request.buildResult(); - String applicationName = effectiveApplicationName(request); - String target = - FlinkDeployMode.isKubernetesMode(request.deployMode()) - ? ", clusterId=" - + request.clusterId() - + ", namespace=" - + request.kubernetesNamespace() - : ""; - AssertUtils.required( - buildResult != null, - "[flink-submit] current job " - + applicationName - + " was not yet built; build result is empty" - + target); - AssertUtils.required( - buildResult.pass(), - "[flink-submit] current job " + applicationName + " build failed" + target); - } - - private static Map getParameters(SubmitRequest request, String prefix) { - if (request.appConf() == null) { - return Collections.emptyMap(); - } - return filterByPrefix(loadApplicationConfig(request.appConf()), prefix); - } - - private static Map loadApplicationConfig(String appConf) { + /** Dispatches job configuration parsing from its explicit transport scheme. */ + private static Map loadJobConfig(String appConf) { if (appConf.length() < CONFIG_SCHEME_LENGTH) { - throw invalidConfigFormat(appConf); + throw createInvalidConfigException(appConf); } String scheme = appConf.substring(0, CONFIG_SCHEME_LENGTH); switch (scheme) { case "json://": - return parseJson(appConf.substring(CONFIG_SCHEME_LENGTH)); + return parseJsonConfig(appConf.substring(CONFIG_SCHEME_LENGTH)); case "yaml://": - return parseConfig(decompress(appConf), ConfigurationFormat.YAML, "inline YAML"); + return parseConfigContent( + decompressConfig(appConf), ConfigurationFormat.YAML, "inline YAML"); case "conf://": - return parseConfig(decompress(appConf), ConfigurationFormat.HOCON, "inline HOCON"); + return parseConfigContent( + decompressConfig(appConf), ConfigurationFormat.HOCON, "inline HOCON"); case "prop://": - return parseConfig( - decompress(appConf), ConfigurationFormat.PROPERTIES, "inline properties"); + return parseConfigContent( + decompressConfig(appConf), ConfigurationFormat.PROPERTIES, "inline properties"); case "hdfs://": - return parseHdfs(appConf); + return parseHdfsConfig(appConf); default: - throw invalidConfigFormat(appConf); + throw createInvalidConfigException(appConf); } } - private static String decompress(String appConf) { + /** Expands a compressed inline configuration after removing its transport scheme. */ + private static String decompressConfig(String appConf) { return DeflaterUtils.unzipString(appConf.trim().substring(CONFIG_SCHEME_LENGTH)); } - private static Map parseJson(String json) { + /** Parses the legacy JSON transport form and discards null entries. */ + private static Map parseJsonConfig(String json) { try { Map values = new ObjectMapper() @@ -243,55 +206,46 @@ private static Map parseJson(String json) { .filter(entry -> entry.getValue() != null) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } catch (Exception e) { - throw new IllegalArgumentException("Failed to parse JSON application configuration", e); + throw new IllegalArgumentException("Failed to parse JSON job configuration", e); } } - private static Map parseHdfs(String appConf) { + /** Reads an HDFS configuration and selects its parser from the file extension. */ + private static Map parseHdfsConfig(String appConf) { try { String text = HdfsUtils.read(appConf); String extension = appConf.substring(appConf.lastIndexOf('.') + 1).toLowerCase(); switch (extension) { case "yml": case "yaml": - return parseConfig(text, ConfigurationFormat.YAML, appConf); + return parseConfigContent(text, ConfigurationFormat.YAML, appConf); case "conf": - return parseConfig(text, ConfigurationFormat.HOCON, appConf); + return parseConfigContent(text, ConfigurationFormat.HOCON, appConf); case "properties": - return parseConfig(text, ConfigurationFormat.PROPERTIES, appConf); + return parseConfigContent(text, ConfigurationFormat.PROPERTIES, appConf); default: throw new IllegalArgumentException( - "HDFS application configuration must be YAML, HOCON, or properties"); + "HDFS job configuration must be YAML, HOCON, or properties"); } } catch (IllegalArgumentException e) { throw e; } catch (Exception e) { throw new IllegalStateException( - "Failed to read HDFS application configuration: " + appConf, e); + "Failed to read HDFS job configuration: " + appConf, e); } } - private static Map parseConfig( - String content, - ConfigurationFormat format, - String origin) { + /** Parses a configuration document through the common immutable configuration pipeline. */ + private static Map parseConfigContent( + String content, + ConfigurationFormat format, + String origin) { return ConfigurationParser.parse(content, format, origin).toMap(); } - private static Map filterByPrefix( - Map values, String prefix) { - Map result = new HashMap<>(); - for (Map.Entry entry : values.entrySet()) { - String value = entry.getValue(); - if (entry.getKey().startsWith(prefix) && value != null && !value.isEmpty()) { - result.put(entry.getKey().substring(prefix.length()), value); - } - } - return result; - } - - private static IllegalArgumentException invalidConfigFormat(String appConf) { + /** Creates a consistent failure for unsupported or truncated transport schemes. */ + private static IllegalArgumentException createInvalidConfigException(String appConf) { return new IllegalArgumentException( - "Unsupported application configuration format: " + appConf); + "Unsupported job configuration format: " + appConf); } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmissionConfigurationBuilder.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationBuilder.java similarity index 56% rename from streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmissionConfigurationBuilder.java rename to streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationBuilder.java index e425792089..fc5849971d 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/bean/SubmissionConfigurationBuilder.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationBuilder.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.streampark.flink.client.bean; +package org.apache.streampark.flink.client.configuration; import org.apache.streampark.common.configuration.CommandLineParser; import org.apache.streampark.common.configuration.CommandLineTokenizer; @@ -32,11 +32,10 @@ import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.ClassLoaderUtils; import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.FlinkConfigurationUtils; +import org.apache.streampark.common.util.FlinkConfigurationLoader; import org.apache.streampark.common.util.LoggerSupport; import org.apache.streampark.common.util.SystemPropertyUtils; -import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; -import org.apache.streampark.flink.client.configuration.FlinkSavepointOptions; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; import org.apache.streampark.flink.client.request.SubmitRequest; import org.apache.commons.cli.CommandLine; @@ -71,11 +70,18 @@ import java.util.Map; import java.util.Set; -/** Builds the effective Flink configuration for a submission request. */ -public final class SubmissionConfigurationBuilder extends LoggerSupport { +/** + * Builds the effective Flink configuration for a submission request. + * + *

Configuration is assembled from one installation snapshot and Flink's own command-line + * implementations. Installation defaults are applied first, request properties and CLI options + * override them, and deployment-specific clients add their platform settings afterward. Keeping + * these stages explicit avoids version-dependent option parsing outside Flink itself. + */ +public final class FlinkConfigurationBuilder extends LoggerSupport { - private static final SubmissionConfigurationBuilder INSTANCE = - new SubmissionConfigurationBuilder(); + private static final FlinkConfigurationBuilder INSTANCE = + new FlinkConfigurationBuilder(); private static final String PARAM_KEY_FLINK_CONF = CommandLineParser.LONG_OPTION_PREFIX + FlinkOptions.FLINK_CONFIGURATION.key(); @@ -91,64 +97,77 @@ public final class SubmissionConfigurationBuilder extends LoggerSupport { private static final String PARAM_KEY_FLINK_PARALLELISM = CommandLineParser.LONG_OPTION_PREFIX + FlinkOptions.PARALLELISM.key(); - private SubmissionConfigurationBuilder() { + private FlinkConfigurationBuilder() { } - /** Builds the configuration used by a deployment client. */ - public static Configuration build(SubmitRequest request) throws Exception { - return INSTANCE.buildConfiguration(request); + /** + * Builds the base configuration used by a deployment client. + * + * @param resolved request values resolved from one immutable configuration snapshot + * @return effective base configuration before platform-specific settings + */ + public static Configuration build(ResolvedSubmitRequest resolved) throws Exception { + return INSTANCE.buildConfiguration(resolved); } - /** Extracts a Flink configuration from dynamic cluster properties. */ + /** + * Extracts a Flink configuration from dynamic cluster properties. + * + * @param flinkHome registered Flink installation directory + * @param properties dynamic properties that override installation defaults + * @return configuration interpreted by the target Flink command line + */ public static Configuration extract(String flinkHome, Map properties) throws Exception { return INSTANCE.extractConfiguration(flinkHome, properties); } - /** Loads the registered Flink installation's default configuration. */ + /** + * Loads the registered Flink installation's default configuration. + * + * @param flinkHome registered Flink installation directory + * @return a new mutable Flink configuration populated from the compatible YAML loader + */ public static Configuration loadDefault(String flinkHome) { - try { - return Configuration.fromMap(FlinkConfigurationUtils.loadFlinkHome(flinkHome)); - } catch (Exception ignored) { - return new Configuration(); - } + return Configuration.fromMap(FlinkConfigurationLoader.loadFlinkHome(flinkHome)); } - /** Resolves the effective parallelism for the request. */ - public static int parallelism(SubmitRequest request) { - Object configured = request.properties().get(FlinkOptions.PARALLELISM.key()); - if (configured != null) { - return Integer.parseInt(configured.toString()); - } - return loadDefault(request.flinkVersion().getFlinkHome()) - .get(CoreOptions.DEFAULT_PARALLELISM, CoreOptions.DEFAULT_PARALLELISM.defaultValue()); - } - - /** Reads an option from the registered Flink installation. */ - public static T getDefaultOption(String flinkHome, ConfigOption option) { + /** + * Reads a typed option from the registered Flink installation. + * + * @param flinkHome registered Flink installation directory + * @param option Flink option to read + * @param option value type + * @return configured or default option value + */ + public static T loadDefaultOption(String flinkHome, ConfigOption option) { return loadDefault(flinkHome).get(option); } - private Configuration buildConfiguration(SubmitRequest request) throws Exception { - CommandLineAndConfiguration cli = parseCommandLine(request); + /** Applies configuration stages in precedence order to one mutable Flink snapshot. */ + private Configuration buildConfiguration(ResolvedSubmitRequest resolved) throws Exception { + SubmitRequest request = resolved.request(); + CommandLineAndConfiguration cli = parseCommandLine(resolved); Configuration configuration = cli.configuration; - applyJobTypeConfiguration(request, cli.commandLine, configuration); - applyPipelineConfiguration(request, configuration); + applyJobTypeConfiguration(resolved, cli.commandLine, configuration); + applyPipelineConfiguration(resolved, configuration); applyCheckpointDefaults(request, configuration); - applySavepointConfiguration(request, configuration); + applySavepointConfiguration(resolved, configuration); applyEnvironmentProperties(request, configuration); return configuration; } + /** Applies the entry point and artifact metadata required by the submitted job type. */ private void applyJobTypeConfiguration( - SubmitRequest request, + ResolvedSubmitRequest resolved, CommandLine commandLine, Configuration configuration) throws Exception { + SubmitRequest request = resolved.request(); if (request.jobType() == FlinkJobType.PYFLINK) { applyPyFlinkConfiguration(request, configuration); return; } - File userJar = SubmitRequestResolver.userJarFile(request); + File userJar = resolved.getUserJarFile(); if (userJar == null) { return; } @@ -159,18 +178,16 @@ private void applyJobTypeConfiguration( .applyToConfiguration(configuration); } + /** Configures the managed Python environment used to launch PyFlink applications. */ private void applyPyFlinkConfiguration( SubmitRequest request, Configuration configuration) throws Exception { String pythonVenv = Workspace.LOCAL.pythonVenv; AssertUtils.required( FsOperator.lfs().exists(pythonVenv), pythonVenv + " file does not exist"); - FlinkConfigurationOps.setIfPresent( - configuration, PythonOptions.PYTHON_ARCHIVES, pythonVenv); - FlinkConfigurationOps.setIfPresent( - configuration, PythonOptions.PYTHON_CLIENT_EXECUTABLE, Constants.PYTHON_EXECUTABLE); - FlinkConfigurationOps.setIfPresent( - configuration, PythonOptions.PYTHON_EXECUTABLE, Constants.PYTHON_EXECUTABLE); + configuration.set(PythonOptions.PYTHON_ARCHIVES, pythonVenv); + configuration.set(PythonOptions.PYTHON_CLIENT_EXECUTABLE, Constants.PYTHON_EXECUTABLE); + configuration.set(PythonOptions.PYTHON_EXECUTABLE, Constants.PYTHON_EXECUTABLE); if (StringUtils.isBlank(System.getenv(ConfigConstants.ENV_FLINK_OPT_DIR))) { String flinkOptPath = request.flinkVersion().getFlinkHome() + "/opt"; @@ -183,44 +200,48 @@ private void applyPyFlinkConfiguration( } } + /** Applies job identity, arguments, deployment target, and fixed job ID. */ private void applyPipelineConfiguration( - SubmitRequest request, Configuration configuration) { - FlinkConfigurationOps.setIfPresent( - configuration, PipelineOptions.NAME, SubmitRequestResolver.effectiveApplicationName(request)); - FlinkConfigurationOps.setIfPresent( - configuration, DeploymentOptions.TARGET, request.deployMode().getName()); - FlinkConfigurationOps.setIfPresent( - configuration, FlinkSavepointOptions.SAVEPOINT_PATH, request.savePoint()); - FlinkConfigurationOps.setIfPresent( - configuration, - ApplicationConfiguration.APPLICATION_MAIN_CLASS, - SubmitRequestResolver.applicationMain(request)); - FlinkConfigurationOps.setIfPresent( - configuration, ApplicationConfiguration.APPLICATION_ARGS, programArguments(request)); - FlinkConfigurationOps.setIfPresent( - configuration, PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, request.jobId()); + ResolvedSubmitRequest resolved, + Configuration configuration) { + SubmitRequest request = resolved.request(); + configuration.set(PipelineOptions.NAME, resolved.getJobName()); + configuration.set(DeploymentOptions.TARGET, request.deployMode().getName()); + + String jobMainClass = resolved.getJobMainClass(); + if (StringUtils.isNotEmpty(jobMainClass)) { + configuration.set(ApplicationConfiguration.APPLICATION_MAIN_CLASS, jobMainClass); + } + + configuration.set(ApplicationConfiguration.APPLICATION_ARGS, programArguments(resolved, configuration)); + if (StringUtils.isNotEmpty(request.jobId())) { + configuration.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, request.jobId()); + } } + /** Materializes the retained-checkpoint default when no dynamic override was supplied. */ private void applyCheckpointDefaults( SubmitRequest request, Configuration configuration) { ConfigOption retainedCheckpoints = CheckpointingOptions.MAX_RETAINED_CHECKPOINTS; if (!request.properties().containsKey(retainedCheckpoints.key())) { configuration.set( retainedCheckpoints, - loadDefault(request.flinkVersion().getFlinkHome()).get(retainedCheckpoints)); + configuration.get(retainedCheckpoints)); } } + /** Applies savepoint restoration only when the request carries a restore path. */ private void applySavepointConfiguration( - SubmitRequest request, Configuration configuration) { + ResolvedSubmitRequest resolved, + Configuration configuration) { + SubmitRequest request = resolved.request(); if (StringUtils.isBlank(request.savePoint())) { return; } - FlinkConfigurationOps.setIfPresent( - configuration, FlinkSavepointOptions.SAVEPOINT_PATH, request.savePoint()); + configuration.set(FlinkSavepointOptions.SAVEPOINT_PATH, request.savePoint()); configuration.set( FlinkSavepointOptions.SAVEPOINT_IGNORE_UNCLAIMED_STATE, - SubmitRequestResolver.allowNonRestoredState(request)); + resolved.isNonRestoredStateAllowed()); if (request.restoreMode() != null && request.flinkVersion().checkVersion(FlinkRestoreMode.SINCE_FLINK_VERSION)) { @@ -228,6 +249,7 @@ private void applySavepointConfiguration( } } + /** Applies environment-prefixed options and the default JVM file encoding. */ private void applyEnvironmentProperties( SubmitRequest request, Configuration configuration) { if (MapUtils.isEmpty(request.properties())) { @@ -243,30 +265,83 @@ private void applyEnvironmentProperties( request.properties().forEach( (key, value) -> { - if (key.startsWith("env.")) { + if (key.startsWith("env.") && value != null) { logInfo("Environment option: " + key + "=" + value); configuration.setString(key, value.toString()); } }); } - private CommandLineAndConfiguration parseCommandLine(SubmitRequest request) throws Exception { + /** + * Parses the request through Flink's own CLI using one default-configuration snapshot. + * + *

Reusing the command-line list and defaults prevents a concurrent file change from + * producing different option discovery and effective configuration in the same submission. + */ + private CommandLineAndConfiguration parseCommandLine(ResolvedSubmitRequest resolved) throws Exception { + SubmitRequest request = resolved.request(); String flinkHome = request.flinkVersion().getFlinkHome(); - Options options = commandLineOptions(flinkHome); - List arguments = cliArguments(request, buildOptionMap(request, options)); + Configuration defaults = loadDefault(flinkHome); + expandJobPlaceholders(defaults, resolved); + List commandLines = customCommandLines(flinkHome, defaults); + Options options = commandLineOptions(commandLines); + List arguments = cliArguments(request, buildOptionMap(resolved, options)); logInfo("Flink CLI arguments: " + String.join(" ", arguments)); CommandLine commandLine = FlinkRunOption.parse(options, arguments.toArray(new String[0]), true); CustomCommandLine activeCommandLine = - activeCommandLine(customCommandLines(flinkHome), commandLine); + activeCommandLine(commandLines, commandLine); return new CommandLineAndConfiguration( - commandLine, applyConfiguration(flinkHome, activeCommandLine, commandLine)); + commandLine, applyConfiguration(defaults, activeCommandLine, commandLine)); } - private Map buildOptionMap(SubmitRequest request, Options commandLineOptions) { + /** + * Replaces StreamPark job variables in deployment-scoped Flink defaults. + * + *

Session and remote clusters retain their shared installation configuration. Other modes + * create job-scoped runtimes and may safely resolve job name and ID placeholders. + */ + static void expandJobPlaceholders( + Configuration configuration, + ResolvedSubmitRequest resolved) { + FlinkDeployMode deployMode = resolved.request().deployMode(); + if (deployMode == FlinkDeployMode.REMOTE + || deployMode == FlinkDeployMode.YARN_SESSION + || deployMode == FlinkDeployMode.KUBERNETES_NATIVE_SESSION) { + return; + } + + String jobName = resolved.getJobName(); + String jobId = Long.toString(resolved.request().id()); + for (String key : configuration.keySet()) { + String value = configuration.getString(key, null); + if (value != null) { + configuration.setString(key, replaceJobPlaceholder(value, jobName, jobId)); + } + } + } + + /** Replaces the historical braced and unbraced spellings of job variables. */ + private static String replaceJobPlaceholder(String value, String jobName, String jobId) { + return value + .replace("${jobName}", jobName) + .replace("${jobname}", jobName) + .replace("$jobName", jobName) + .replace("$jobname", jobName) + .replace("${jobId}", jobId) + .replace("${jobid}", jobId) + .replace("$jobId", jobId) + .replace("$jobid", jobId); + } + + /** Filters persisted job options against the options supported by the target Flink. */ + private Map buildOptionMap( + ResolvedSubmitRequest resolved, + Options commandLineOptions) { + SubmitRequest request = resolved.request(); Map options = new HashMap<>(); - SubmitRequestResolver.applicationOptions(request).forEach( + resolved.jobOptions().forEach( (key, value) -> { if (!commandLineOptions.hasOption(key)) { logWarn("Ignoring unsupported Flink CLI option: " + key); @@ -294,6 +369,7 @@ private Map buildOptionMap(SubmitRequest request, Options comman return options; } + /** Converts validated options and dynamic properties to Flink CLI arguments. */ private List cliArguments(SubmitRequest request, Map options) { List arguments = new ArrayList<>(); options.forEach( @@ -312,12 +388,16 @@ private List cliArguments(SubmitRequest request, Map opt return arguments; } - private List programArguments(SubmitRequest request) { + /** Builds the argument vector delivered to the submitted job's main method. */ + private List programArguments( + ResolvedSubmitRequest resolved, + Configuration configuration) { + SubmitRequest request = resolved.request(); List arguments = new ArrayList<>(CommandLineTokenizer.tokenize(request.args())); if (request.applicationType() == ApplicationType.STREAMPARK_FLINK || request.jobType() == FlinkJobType.FLINK_SQL) { - addStreamParkArguments(request, arguments); + addStreamParkArguments(resolved, configuration, arguments); } Object runtimeMode = request.properties().get(ExecutionOptions.RUNTIME_MODE.key()); @@ -328,7 +408,7 @@ private List programArguments(SubmitRequest request) { if (request.jobType() == FlinkJobType.PYFLINK && request.deployMode() != FlinkDeployMode.YARN_APPLICATION) { - File userJar = SubmitRequestResolver.userJarFile(request); + File userJar = resolved.getUserJarFile(); AssertUtils.notNull(userJar); arguments.add("-py"); arguments.add(userJar.getAbsolutePath()); @@ -336,7 +416,12 @@ private List programArguments(SubmitRequest request) { return arguments; } - private void addStreamParkArguments(SubmitRequest request, List arguments) { + /** Adds StreamPark transport arguments consumed by its Flink runtime entry points. */ + private void addStreamParkArguments( + ResolvedSubmitRequest resolved, + Configuration configuration, + List arguments) { + SubmitRequest request = resolved.request(); arguments.add(PARAM_KEY_FLINK_CONF); arguments.add(request.flinkYaml()); // Carry the persisted syntax snapshot so Flink 1.19 never re-resolves its configuration @@ -344,42 +429,54 @@ private void addStreamParkArguments(SubmitRequest request, List argument arguments.add(PARAM_KEY_FLINK_CONF_STANDARD_YAML); arguments.add(Boolean.toString(request.standardYaml())); arguments.add(PARAM_KEY_APP_NAME); - arguments.add( - DeflaterUtils.zipString(SubmitRequestResolver.effectiveApplicationName(request))); + arguments.add(DeflaterUtils.zipString(resolved.getJobName())); arguments.add(PARAM_KEY_FLINK_PARALLELISM); - arguments.add(Integer.toString(parallelism(request))); + arguments.add( + Integer.toString( + configuration.get( + CoreOptions.DEFAULT_PARALLELISM, + CoreOptions.DEFAULT_PARALLELISM.defaultValue()))); if (request.jobType() == FlinkJobType.FLINK_SQL) { arguments.add(PARAM_KEY_FLINK_SQL); - arguments.add(SubmitRequestResolver.flinkSql(request)); - addApplicationConfiguration(request.appConf(), arguments); - } else if (shouldAddApplicationConfiguration(request.appConf())) { - addApplicationConfiguration(request.appConf(), arguments); + arguments.add(resolved.getFlinkSqlContent()); + addJobConfiguration(request.appConf(), arguments); + } else if (shouldAddJobConfiguration(request.appConf())) { + addJobConfiguration(request.appConf(), arguments); } } - private void addApplicationConfiguration(String appConf, List arguments) { + /** Appends a non-null serialized job configuration argument. */ + private void addJobConfiguration(String appConf, List arguments) { if (appConf != null) { arguments.add(PARAM_KEY_APP_CONF); arguments.add(appConf); } } + /** Builds a Flink configuration from dynamic properties without a job request. */ private Configuration extractConfiguration( String flinkHome, Map properties) throws Exception { + Configuration defaults = loadDefault(flinkHome); + List commandLines = customCommandLines(flinkHome, defaults); List arguments = new ArrayList<>(); if (MapUtils.isNotEmpty(properties)) { properties.forEach( - (key, value) -> arguments.add("-D" + key + "=" + value.toString().trim())); + (key, value) -> { + if (value != null) { + arguments.add("-D" + key + "=" + value.toString().trim()); + } + }); } CommandLine commandLine = FlinkRunOption.parse( - commandLineOptions(flinkHome), arguments.toArray(new String[0]), true); + commandLineOptions(commandLines), arguments.toArray(new String[0]), true); CustomCommandLine activeCommandLine = - activeCommandLine(customCommandLines(flinkHome), commandLine); - return applyConfiguration(flinkHome, activeCommandLine, commandLine); + activeCommandLine(commandLines, commandLine); + return applyConfiguration(defaults, activeCommandLine, commandLine); } + /** Selects the first target-specific command line that accepts the parsed arguments. */ private CustomCommandLine activeCommandLine( List commandLines, CommandLine commandLine) { @@ -392,18 +489,21 @@ private CustomCommandLine activeCommandLine( throw new IllegalStateException("No active Flink command line found"); } - private List customCommandLines(String flinkHome) { - Configuration defaultConfiguration = loadDefault(flinkHome); + /** Loads Flink's built-in and plugin command lines against the captured defaults. */ + private List customCommandLines( + String flinkHome, + Configuration defaults) { String configurationDirectory = flinkHome + "/conf"; return ClassLoaderUtils.runAsClassLoader( - SubmissionConfigurationBuilder.class.getClassLoader(), + FlinkConfigurationBuilder.class.getClassLoader(), () -> CliFrontend.loadCustomCommandLines( - defaultConfiguration, configurationDirectory)); + defaults, configurationDirectory)); } - private Options commandLineOptions(String flinkHome) { + /** Merges target-specific CLI options with Flink's standard run options. */ + private Options commandLineOptions(List commandLines) { Options customOptions = new Options(); - for (CustomCommandLine commandLine : customCommandLines(flinkHome)) { + for (CustomCommandLine commandLine : commandLines) { commandLine.addGeneralOptions(customOptions); commandLine.addRunOptions(customOptions); } @@ -411,13 +511,13 @@ private Options commandLineOptions(String flinkHome) { CliFrontendParser.getRunCommandOptions(), customOptions); } + /** Applies captured defaults first and dynamic CLI configuration last. */ private Configuration applyConfiguration( - String flinkHome, + Configuration defaults, CustomCommandLine activeCommandLine, CommandLine commandLine) throws Exception { Preconditions.checkNotNull(activeCommandLine, "activeCommandLine must not be null"); Configuration configuration = new Configuration(); - Configuration defaults = loadDefault(flinkHome); Set keys = defaults.keySet(); for (String key : keys) { String value = defaults.getString(key, null); @@ -429,6 +529,7 @@ private Configuration applyConfiguration( return configuration; } + /** Preserves boolean flags as booleans while leaving valued options as strings. */ private static Object parseOptionValue(String value) { if ("true".equalsIgnoreCase(value) || "false".equalsIgnoreCase(value)) { return Boolean.parseBoolean(value); @@ -436,7 +537,8 @@ private static Object parseOptionValue(String value) { return value; } - private static boolean shouldAddApplicationConfiguration(String appConf) { + /** Determines whether job configuration must be forwarded to the runtime. */ + private static boolean shouldAddJobConfiguration(String appConf) { return StringUtils.isNotBlank(appConf) && !appConf.startsWith("json:"); } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationOps.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationOps.java deleted file mode 100644 index 93cdec3b0c..0000000000 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationOps.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.streampark.flink.client.configuration; - -import org.apache.flink.configuration.ConfigOption; -import org.apache.flink.configuration.Configuration; - -/** Null-safe operations for Flink configuration values. */ -public final class FlinkConfigurationOps { - - private FlinkConfigurationOps() { - } - - /** Sets a non-null, non-empty value and leaves the configuration unchanged otherwise. */ - public static Configuration setIfPresent( - Configuration flinkConfig, - ConfigOption option, - T value) { - if (value != null && !value.toString().isEmpty()) { - flinkConfig.set(option, value); - } - return flinkConfig; - } -} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java index bf2f0fe312..a296fb48e1 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractFlinkClient.java @@ -24,12 +24,13 @@ import org.apache.streampark.common.util.LoggerSupport; import org.apache.streampark.common.util.Utils; import org.apache.streampark.flink.client.bean.FlinkJobGraphBuilder; -import org.apache.streampark.flink.client.bean.SubmissionConfigurationBuilder; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.configuration.FlinkConfigurationBuilder; +import org.apache.streampark.flink.client.request.AbstractSavepointRequest; import org.apache.streampark.flink.client.request.CancelRequest; import org.apache.streampark.flink.client.request.SavepointRequest; import org.apache.streampark.flink.client.request.SubmitRequest; -import org.apache.streampark.flink.client.request.TriggerSavepointRequest; import org.apache.streampark.flink.client.response.CancelResponse; import org.apache.streampark.flink.client.response.SavepointResponse; import org.apache.streampark.flink.client.response.SubmitResponse; @@ -40,7 +41,6 @@ import org.apache.flink.api.common.JobID; import org.apache.flink.client.cli.CliArgsException; import org.apache.flink.client.program.ClusterClient; -import org.apache.flink.client.program.PackagedProgram; import org.apache.flink.configuration.CheckpointingOptions; import org.apache.flink.configuration.ConfigOptions; import org.apache.flink.configuration.Configuration; @@ -60,39 +60,41 @@ public abstract class AbstractFlinkClient extends LoggerSupport { /** Submits a job with the deployment-mode-specific client. */ public final SubmitResponse submit(SubmitRequest request) throws FlinkException { - logSubmitRequest(request); - Configuration configuration = - callAsFlinkException(() -> SubmissionConfigurationBuilder.build(request)); - setConfig(request, configuration); - return callAsFlinkException( - () -> doSubmit(request, configuration), - error -> logSubmitFailure(request, error)); + ResolvedSubmitRequest resolved = execute(() -> SubmitRequestResolver.resolve(request)); + logSubmitRequest(resolved); + Configuration configuration = execute(() -> FlinkConfigurationBuilder.build(resolved)); + setConfig(resolved, configuration); + return execute( + () -> doSubmit(resolved, configuration), + error -> logSubmitFailure(resolved, error)); } /** Triggers a savepoint for an existing job. */ - public final SavepointResponse triggerSavepoint(TriggerSavepointRequest request) throws FlinkException { + public final SavepointResponse triggerSavepoint(SavepointRequest request) throws FlinkException { logSavepointRequest("trigger savepoint", request); - return callAsFlinkException( + return execute( () -> doTriggerSavepoint(request, new Configuration())); } /** Cancels an existing job, optionally creating a savepoint first. */ public final CancelResponse cancel(CancelRequest request) throws FlinkException { logSavepointRequest("cancel", request); - return callAsFlinkException(() -> doCancel(request, new Configuration())); + return execute(() -> doCancel(request, new Configuration())); } /** Adds configuration required by a deployment mode. */ - protected abstract void setConfig(SubmitRequest request, Configuration configuration); + protected abstract void setConfig( + ResolvedSubmitRequest resolved, + Configuration configuration); /** Performs the deployment-mode-specific submission. */ protected abstract SubmitResponse doSubmit( - SubmitRequest request, + ResolvedSubmitRequest resolved, Configuration configuration) throws FlinkException; /** Performs the deployment-mode-specific savepoint operation. */ protected abstract SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest request, + SavepointRequest request, Configuration configuration) throws FlinkException; /** Performs the deployment-mode-specific cancellation. */ @@ -101,7 +103,7 @@ protected abstract CancelResponse doCancel( Configuration configuration) throws FlinkException; /** Converts infrastructure exceptions to the stable Flink client exception contract. */ - protected static FlinkException asFlinkException(Throwable throwable) { + protected static FlinkException mapException(Throwable throwable) { if (throwable instanceof FlinkException) { return (FlinkException) throwable; } @@ -111,14 +113,16 @@ protected static FlinkException asFlinkException(Throwable throwable) { return new FlinkException(throwable.getMessage(), throwable); } - protected static T callAsFlinkException(FlinkCallable callable) throws FlinkException { - return callAsFlinkException(callable, ignored -> { + /** Executes an operation while preserving the public {@link FlinkException} contract. */ + protected static T execute(ClientOperation callable) throws FlinkException { + return execute(callable, ignored -> { }); } - protected static T callAsFlinkException( - FlinkCallable callable, - Consumer onFailure) throws FlinkException { + /** Executes an operation and invokes the failure callback before exception conversion. */ + protected static T execute( + ClientOperation callable, + Consumer onFailure) throws FlinkException { try { return callable.call(); } catch (FlinkException e) { @@ -126,13 +130,14 @@ protected static T callAsFlinkException( throw e; } catch (Exception e) { onFailure.accept(e); - throw asFlinkException(e); + throw mapException(e); } } - protected static T callAsFlinkExceptionMapping( - FlinkCallable callable, - Function exceptionMapper) throws FlinkException { + /** Executes an operation with caller-defined conversion for non-Flink exceptions. */ + protected static T executeAndMapError( + ClientOperation callable, + Function exceptionMapper) throws FlinkException { try { return callable.call(); } catch (FlinkException e) { @@ -142,33 +147,42 @@ protected static T callAsFlinkExceptionMapping( } } + /** Cancels a job and wraps its optional savepoint path in the stable response type. */ protected final CancelResponse toCancelResponse( CancelRequest request, JobID jobId, ClusterClient client) throws FlinkException { - return callAsFlinkException(() -> new CancelResponse(cancelJob(request, jobId, client))); + return execute(() -> new CancelResponse(cancelJob(request, jobId, client))); } + /** Triggers a savepoint and wraps its path in the stable response type. */ protected final SavepointResponse toSavepointResponse( - TriggerSavepointRequest request, + SavepointRequest request, JobID jobId, ClusterClient client) throws FlinkException { - return callAsFlinkException( + return execute( () -> new SavepointResponse(triggerSavepoint(request, jobId, client))); } + /** + * Builds and submits a JobGraph while owning every resource opened for the submission. + * + *

The packaged program, cluster client, and supplied descriptors are closed on both success + * and failure. Callers must not reuse the supplied resources after this method returns. + */ protected final SubmitResponse submitJobGraphToCluster( - SubmitRequest request, + ResolvedSubmitRequest resolved, Configuration configuration, File jarFile, - FlinkCallable> clientSupplier, - FlinkCallable clusterIdSupplier, + ClientOperation> clientSupplier, + ClientOperation clusterIdSupplier, AutoCloseable... extraResources) throws FlinkException { - return callAsFlinkException( + return execute( () -> { - FlinkJobGraphBuilder.Result result = buildJobGraph(configuration, request, jarFile); + FlinkJobGraphBuilder.Result result = null; ClusterClient client = null; try { + result = buildJobGraph(configuration, resolved, jarFile); client = clientSupplier.call(); String jobId = submitJobGraph(client, result.jobGraph()); return new SubmitResponse( @@ -178,30 +192,30 @@ protected final SubmitResponse submitJobGraphToCluster( client.getWebInterfaceURL()); } finally { AutoCloseable[] resources = new AutoCloseable[extraResources.length + 2]; - resources[0] = result.program(); + resources[0] = result == null ? null : result.program(); resources[1] = client; System.arraycopy(extraResources, 0, resources, 2, extraResources.length); - closeSubmissionResources(request, resources); + closeSubmissionResources(resources); } }); } /** Tries JobGraph submission before falling back to the session REST API. */ protected final SubmitResponse trySubmit( - SubmitRequest request, + ResolvedSubmitRequest resolved, Configuration configuration, File jarFile, - SubmitFunction jobGraphSubmit, - SubmitFunction restApiSubmit) throws FlinkException { + SubmissionStrategy jobGraphSubmit, + SubmissionStrategy restApiSubmit) throws FlinkException { try { logInfo("[flink-submit] Submitting with the JobGraph protocol"); - return jobGraphSubmit.apply(request, configuration, jarFile); + return jobGraphSubmit.apply(resolved, configuration, jarFile); } catch (FlinkException jobGraphFailure) { logWarn( "[flink-submit] JobGraph submission failed; trying the REST API: " + ExceptionUtils.stringifyException(jobGraphFailure)); try { - return restApiSubmit.apply(request, configuration, jarFile); + return restApiSubmit.apply(resolved, configuration, jarFile); } catch (FlinkException restFailure) { restFailure.addSuppressed(jobGraphFailure); throw new FlinkException( @@ -211,13 +225,15 @@ protected final SubmitResponse trySubmit( } } + /** Delegates JobGraph construction to the version-aware builder. */ protected final FlinkJobGraphBuilder.Result buildJobGraph( Configuration configuration, - SubmitRequest request, + ResolvedSubmitRequest resolved, File jarFile) throws Exception { - return FlinkJobGraphBuilder.build(configuration, request, jarFile); + return FlinkJobGraphBuilder.build(configuration, resolved, jarFile); } + /** Parses a hexadecimal job ID and reports invalid input as a CLI argument error. */ protected final JobID parseJobId(String jobId) throws CliArgsException { try { return JobID.fromHexString(jobId); @@ -226,16 +242,19 @@ protected final JobID parseJobId(String jobId) throws CliArgsException { } } + /** Extracts dynamic cluster properties through Flink's target-specific command line. */ protected final Configuration extractConfiguration( String flinkHome, Map properties) throws Exception { - return SubmissionConfigurationBuilder.extract(flinkHome, properties); + return FlinkConfigurationBuilder.extract(flinkHome, properties); } + /** Loads the registered Flink installation's default configuration. */ protected final Configuration loadDefaultConfiguration(String flinkHome) { - return SubmissionConfigurationBuilder.loadDefault(flinkHome); + return FlinkConfigurationBuilder.loadDefault(flinkHome); } + /** Cancels a job directly or stops it with a savepoint according to the request. */ protected final String cancelJob( CancelRequest request, JobID jobId, ClusterClient client) throws Exception { String savepointDirectory = resolveSavepointDirectory(request); @@ -249,8 +268,9 @@ protected final String cancelJob( .get(); } + /** Triggers a savepoint with the request's format and resolved target directory. */ protected final String triggerSavepoint( - TriggerSavepointRequest request, + SavepointRequest request, JobID jobId, ClusterClient client) throws Exception { return new FlinkClusterClient<>(client) @@ -259,20 +279,16 @@ jobId, resolveSavepointDirectory(request), request.nativeFormat()) .get(); } - protected final void closeSubmissionResources( - SubmitRequest request, - AutoCloseable... resources) { + /** Closes submission resources independently so one close failure cannot skip later resources. */ + protected final void closeSubmissionResources(AutoCloseable... resources) { for (AutoCloseable resource : resources) { - if (resource == null) { - continue; - } - if (!(resource instanceof PackagedProgram) - || SubmitRequestResolver.canSafelyClosePackagedProgram(request)) { + if (resource != null) { Utils.close(resource); } } } + /** Logs the final Flink configuration after all deployment-specific options are applied. */ protected final void logEffectiveSubmitConfiguration(Configuration configuration) { logInfo( String.format( @@ -282,7 +298,8 @@ protected final void logEffectiveSubmitConfiguration(Configuration configuration configuration)); } - private String resolveSavepointDirectory(SavepointRequest request) { + /** Resolves the explicit or installation-level savepoint directory for a job operation. */ + private String resolveSavepointDirectory(AbstractSavepointRequest request) { if (!request.withSavepoint()) { return null; } @@ -291,7 +308,7 @@ private String resolveSavepointDirectory(SavepointRequest request) { } String defaultDirectory = - SubmissionConfigurationBuilder.getDefaultOption( + FlinkConfigurationBuilder.loadDefaultOption( request.flinkVersion().getFlinkHome(), ConfigOptions.key(CheckpointingOptions.SAVEPOINT_DIRECTORY.key()) .stringType() @@ -325,7 +342,9 @@ private static String submitJobGraph(ClusterClient client, JobGraph jobGraph) + client.getClass().getName()); } - private void logSubmitRequest(SubmitRequest request) { + /** Logs a submission request without relying on mutable map iteration at call sites. */ + private void logSubmitRequest(ResolvedSubmitRequest resolved) { + SubmitRequest request = resolved.request(); logInfo( String.format( "%n--------------------------------------- flink job start ---------------------------------------%n" @@ -346,7 +365,7 @@ private void logSubmitRequest(SubmitRequest request) { + "-------------------------------------------------------------------------------------------%n", request.flinkVersion().getFlinkHome(), request.flinkVersion().version(), - SubmitRequestResolver.effectiveApplicationName(request), + resolved.getJobName(), request.jobType(), request.deployMode(), request.kubernetesNamespace(), @@ -360,17 +379,20 @@ private void logSubmitRequest(SubmitRequest request) { request.buildResult())); } - private void logSubmitFailure(SubmitRequest request, Exception error) { + /** Logs a submission failure with the resolved job and deployment mode. */ + private void logSubmitFailure(ResolvedSubmitRequest resolved, Exception error) { + SubmitRequest request = resolved.request(); logError( "Flink job " - + SubmitRequestResolver.effectiveApplicationName(request) + + resolved.getJobName() + " failed to start in " + request.deployMode().getName() + ": " + ExceptionUtils.stringifyException(error)); } - private void logSavepointRequest(String operation, SavepointRequest request) { + /** Logs the stable identity and savepoint options of a job operation. */ + private void logSavepointRequest(String operation, AbstractSavepointRequest request) { logInfo( "Flink job " + operation @@ -386,6 +408,7 @@ private void logSavepointRequest(String operation, SavepointRequest request) { + request.nativeFormat()); } + /** Formats dynamic properties for diagnostic logging. */ private static String formatProperties(Map properties) { if (MapUtils.isEmpty(properties)) { return ""; @@ -395,15 +418,21 @@ private static String formatProperties(Map properties) { .collect(Collectors.joining(" ")); } + /** Operation that may throw an infrastructure-specific checked exception. */ @FunctionalInterface - protected interface FlinkCallable { + protected interface ClientOperation { T call() throws Exception; } + /** Submission strategy used by the JobGraph-to-REST fallback chain. */ @FunctionalInterface - protected interface SubmitFunction { + protected interface SubmissionStrategy { - SubmitResponse apply(SubmitRequest request, Configuration configuration, File jarFile) throws FlinkException; + SubmitResponse apply( + ResolvedSubmitRequest resolved, + Configuration configuration, + File jarFile) throws FlinkException; } + } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractKubernetesNativeClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractKubernetesNativeClient.java index 401deab92a..19acd2dbd4 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractKubernetesNativeClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractKubernetesNativeClient.java @@ -20,11 +20,11 @@ import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkKubernetesRestExposedType; import org.apache.streampark.common.util.Tuple2; -import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; +import org.apache.streampark.flink.client.request.AbstractSavepointRequest; import org.apache.streampark.flink.client.request.CancelRequest; import org.apache.streampark.flink.client.request.SavepointRequest; import org.apache.streampark.flink.client.request.SubmitRequest; -import org.apache.streampark.flink.client.request.TriggerSavepointRequest; import org.apache.streampark.flink.client.response.CancelResponse; import org.apache.streampark.flink.client.response.SavepointResponse; import org.apache.streampark.flink.kubernetes.PodTemplateTool; @@ -50,16 +50,21 @@ /** Base implementation for Kubernetes native deployment clients. */ public abstract class AbstractKubernetesNativeClient extends AbstractFlinkClient { + /** Adds Kubernetes identity, exposure, pod-template, and configuration-directory options. */ @Override - protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - FlinkConfigurationOps.setIfPresent( - flinkConfig, KubernetesConfigOptions.CLUSTER_ID, submitRequest.clusterId()); - FlinkConfigurationOps.setIfPresent( - flinkConfig, KubernetesConfigOptions.NAMESPACE, submitRequest.kubernetesNamespace()); - FlinkConfigurationOps.setIfPresent( - flinkConfig, + protected void setConfig(ResolvedSubmitRequest resolved, Configuration flinkConfig) { + SubmitRequest submitRequest = resolved.request(); + + if (StringUtils.isNotEmpty(submitRequest.clusterId())) { + flinkConfig.set(KubernetesConfigOptions.CLUSTER_ID, submitRequest.clusterId()); + } + if (StringUtils.isNotEmpty(submitRequest.kubernetesNamespace())) { + flinkConfig.set( + KubernetesConfigOptions.NAMESPACE, submitRequest.kubernetesNamespace()); + } + flinkConfig.set( KubernetesConfigOptions.REST_SERVICE_EXPOSED_TYPE, - covertToServiceExposedType(submitRequest.flinkRestExposedType())); + convertServiceExposedType(submitRequest.flinkRestExposedType())); if (submitRequest.buildResult() != null && submitRequest.deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) { @@ -68,33 +73,29 @@ protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) Map podTemplatePaths = buildResult.podTemplatePaths(); if (podTemplatePaths != null) { for (Map.Entry entry : podTemplatePaths.entrySet()) { + if (StringUtils.isEmpty(entry.getValue())) { + continue; + } if (PodTemplateTool.KUBERNETES_POD_TEMPLATE.key().equals(entry.getKey())) { - FlinkConfigurationOps.setIfPresent( - flinkConfig, - KubernetesConfigOptions.KUBERNETES_POD_TEMPLATE, - entry.getValue()); + flinkConfig.set( + KubernetesConfigOptions.KUBERNETES_POD_TEMPLATE, entry.getValue()); } else if (PodTemplateTool.KUBERNETES_JM_POD_TEMPLATE .key() .equals(entry.getKey())) { - FlinkConfigurationOps.setIfPresent( - flinkConfig, - KubernetesConfigOptions.JOB_MANAGER_POD_TEMPLATE, - entry.getValue()); + flinkConfig.set( + KubernetesConfigOptions.JOB_MANAGER_POD_TEMPLATE, entry.getValue()); } else if (PodTemplateTool.KUBERNETES_TM_POD_TEMPLATE .key() .equals(entry.getKey())) { - FlinkConfigurationOps.setIfPresent( - flinkConfig, - KubernetesConfigOptions.TASK_MANAGER_POD_TEMPLATE, - entry.getValue()); + flinkConfig.set( + KubernetesConfigOptions.TASK_MANAGER_POD_TEMPLATE, entry.getValue()); } } } } if (!flinkConfig.contains(DeploymentOptionsInternal.CONF_DIR)) { - FlinkConfigurationOps.setIfPresent( - flinkConfig, + flinkConfig.set( DeploymentOptionsInternal.CONF_DIR, submitRequest.flinkVersion().getFlinkHome() + "/conf"); } @@ -106,10 +107,7 @@ protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) logEffectiveSubmitConfiguration(flinkConfig); } - protected void setK8sDeployTarget(Configuration flinkConf, FlinkDeployMode deployMode) { - FlinkConfigurationOps.setIfPresent(flinkConf, DeploymentOptions.TARGET, deployMode.getName()); - } - + /** Cancels a Kubernetes job and removes its application cluster when required. */ @Override protected CancelResponse doCancel( CancelRequest cancelRequest, @@ -118,7 +116,7 @@ protected CancelResponse doCancel( cancelRequest, flinkConfig, (jobId, client) -> { - String resp = callAsFlinkException(() -> cancelJob(cancelRequest, jobId, client)); + String resp = execute(() -> cancelJob(cancelRequest, jobId, client)); if (cancelRequest.deployMode() == FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION) { client.shutDownCluster(); } @@ -126,9 +124,10 @@ protected CancelResponse doCancel( }); } + /** Triggers a savepoint through the target Kubernetes cluster client. */ @Override protected SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest savepointRequest, + SavepointRequest savepointRequest, Configuration flinkConfig) throws FlinkException { return executeClientAction( savepointRequest, @@ -136,8 +135,37 @@ protected SavepointResponse doTriggerSavepoint( (jobId, clusterClient) -> toSavepointResponse(savepointRequest, jobId, clusterClient)); } + /** Creates the descriptor and resource specification used for Kubernetes deployment. */ + protected final Tuple2 createClusterDescriptorAndSpec(Configuration flinkConfig) { + KubernetesClusterClientFactory clientFactory = new KubernetesClusterClientFactory(); + KubernetesClusterDescriptor clusterDescriptor = clientFactory.createClusterDescriptor(flinkConfig); + ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(flinkConfig); + return new Tuple2<>(clusterDescriptor, clusterSpecification); + } + + /** Formats the deployment identity included in Kubernetes submission logs. */ + protected String flinkConfIdentifierInfo(@Nonnull Configuration conf) { + return "deployMode=" + + conf.get(DeploymentOptions.TARGET) + + ", clusterId=" + + conf.get(KubernetesConfigOptions.CLUSTER_ID) + + ", " + + "namespace=" + + conf.get(KubernetesConfigOptions.NAMESPACE); + } + + /** Resolves an explicit kubeconfig path or the current user's default path. */ + protected final String getDefaultKubernetesConf(String k8sConf) { + String homePath = System.getProperty("user.home"); + if (k8sConf != null) { + return k8sConf.replace("~", homePath); + } + return homePath.concat("/.kube/config"); + } + + /** Executes a job action against the cluster identified by the serialized request. */ private O executeClientAction( - SavepointRequest request, + AbstractSavepointRequest request, Configuration flinkConfig, ClientAction actFunc) throws FlinkException { String hints = @@ -149,16 +177,14 @@ private O executeClientAction( + flinkConfig.get(DeploymentOptions.TARGET)); } - FlinkConfigurationOps.setIfPresent( - flinkConfig, KubernetesConfigOptions.CLUSTER_ID, request.clusterId()); - FlinkConfigurationOps.setIfPresent( - flinkConfig, KubernetesConfigOptions.NAMESPACE, request.kubernetesNamespace()); + flinkConfig.set(KubernetesConfigOptions.CLUSTER_ID, request.clusterId()); + flinkConfig.set(KubernetesConfigOptions.NAMESPACE, request.kubernetesNamespace()); KubernetesClusterDescriptor clusterDescriptor = null; ClusterClient client = null; try { - clusterDescriptor = getK8sClusterDescriptor(flinkConfig); + clusterDescriptor = getClusterDescriptor(flinkConfig); client = clusterDescriptor .retrieve( @@ -174,7 +200,7 @@ private O executeClientAction( throw e; } catch (Exception e) { logClientActionFailure(hints, flinkConfig, request, e); - throw asFlinkException(e); + throw mapException(e); } finally { if (client != null) { client.close(); @@ -185,10 +211,11 @@ private O executeClientAction( } } + /** Logs the complete cluster identity when a Kubernetes job action fails. */ private void logClientActionFailure( String hints, Configuration flinkConfig, - SavepointRequest request, + AbstractSavepointRequest request, Exception e) { logError( hints @@ -199,40 +226,8 @@ private void logClientActionFailure( e); } - protected final Tuple2 getK8sClusterDescriptorAndSpecification(Configuration flinkConfig) { - KubernetesClusterClientFactory clientFactory = new KubernetesClusterClientFactory(); - KubernetesClusterDescriptor clusterDescriptor = - clientFactory.createClusterDescriptor(flinkConfig); - ClusterSpecification clusterSpecification = - clientFactory.getClusterSpecification(flinkConfig); - return new Tuple2<>(clusterDescriptor, clusterSpecification); - } - - protected final KubernetesClusterDescriptor getK8sClusterDescriptor( - Configuration flinkConfig) { - KubernetesClusterClientFactory clientFactory = new KubernetesClusterClientFactory(); - return clientFactory.createClusterDescriptor(flinkConfig); - } - - protected String flinkConfIdentifierInfo(@Nonnull Configuration conf) { - return "deployMode=" - + conf.get(DeploymentOptions.TARGET) - + ", clusterId=" - + conf.get(KubernetesConfigOptions.CLUSTER_ID) - + ", " - + "namespace=" - + conf.get(KubernetesConfigOptions.NAMESPACE); - } - - protected final String getDefaultKubernetesConf(String k8sConf) { - String homePath = System.getProperty("user.home"); - if (k8sConf != null) { - return k8sConf.replace("~", homePath); - } - return homePath.concat("/.kube/config"); - } - - private ServiceExposedType covertToServiceExposedType(FlinkKubernetesRestExposedType exposedType) { + /** Maps StreamPark's stable REST exposure enum to the target Flink enum. */ + private ServiceExposedType convertServiceExposedType(FlinkKubernetesRestExposedType exposedType) { if (exposedType == FlinkKubernetesRestExposedType.CLUSTER_IP) { return ServiceExposedType.ClusterIP; } @@ -245,6 +240,13 @@ private ServiceExposedType covertToServiceExposedType(FlinkKubernetesRestExposed return ServiceExposedType.LoadBalancer; } + /** Creates a descriptor for an existing Kubernetes cluster operation. */ + private final KubernetesClusterDescriptor getClusterDescriptor( + Configuration flinkConfig) { + KubernetesClusterClientFactory clientFactory = new KubernetesClusterClientFactory(); + return clientFactory.createClusterDescriptor(flinkConfig); + } + @FunctionalInterface private interface ClientAction { diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractYarnClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractYarnClient.java index 3fc96283b9..5c4f95f8a4 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractYarnClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/AbstractYarnClient.java @@ -21,12 +21,11 @@ import org.apache.streampark.common.util.HadoopUtils; import org.apache.streampark.common.util.Tuple2; import org.apache.streampark.common.util.Utils; -import org.apache.streampark.flink.client.bean.SubmitRequestResolver; -import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; +import org.apache.streampark.flink.client.request.AbstractSavepointRequest; import org.apache.streampark.flink.client.request.CancelRequest; import org.apache.streampark.flink.client.request.SavepointRequest; import org.apache.streampark.flink.client.request.SubmitRequest; -import org.apache.streampark.flink.client.request.TriggerSavepointRequest; import org.apache.streampark.flink.client.response.CancelResponse; import org.apache.streampark.flink.client.response.SavepointResponse; @@ -43,6 +42,8 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.ApplicationId; +import javax.annotation.Nonnull; + import java.lang.reflect.Method; import java.security.PrivilegedAction; @@ -51,22 +52,25 @@ public abstract class AbstractYarnClient extends AbstractFlinkClient { private volatile Method deployInternalMethod; + /** Adds the job identity shared by all YARN deployment modes. */ @Override - protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - FlinkConfigurationOps.setIfPresent( - flinkConfig, + protected void setConfig(ResolvedSubmitRequest resolved, Configuration flinkConfig) { + SubmitRequest submitRequest = resolved.request(); + flinkConfig.set( YarnConfigOptions.APPLICATION_NAME, - SubmitRequestResolver.effectiveApplicationName(submitRequest)); - FlinkConfigurationOps.setIfPresent( - flinkConfig, + resolved.getJobName()); + + flinkConfig.set( YarnConfigOptions.APPLICATION_TYPE, submitRequest.applicationType().getName()); - FlinkConfigurationOps.setIfPresent(flinkConfig, YarnConfigOptions.APPLICATION_TAGS, "streampark"); + + flinkConfig.set(YarnConfigOptions.APPLICATION_TAGS, "streampark"); } + /** Triggers a savepoint through the YARN application identified by the request. */ @Override protected SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest savepointRequest, + SavepointRequest savepointRequest, Configuration flinkConf) throws FlinkException { return executeClientAction( savepointRequest, @@ -74,6 +78,7 @@ protected SavepointResponse doTriggerSavepoint( (jobId, client) -> toSavepointResponse(savepointRequest, jobId, client)); } + /** Cancels a job through the YARN application identified by the request. */ @Override protected CancelResponse doCancel( CancelRequest cancelRequest, @@ -84,11 +89,16 @@ protected CancelResponse doCancel( (jobId, client) -> toCancelResponse(cancelRequest, jobId, client)); } + /** + * Invokes Flink's internal per-job deployment entry point. + * + *

Flink exposes no public API that preserves the required per-job behavior. Reflection is + * isolated here so concrete clients do not depend on the private method directly. + */ @SuppressWarnings("unchecked") - protected final ClusterClientProvider deployInternal( - YarnClusterDescriptor clusterDescriptor, + protected final ClusterClientProvider deployInternal(YarnClusterDescriptor clusterDescriptor, ClusterSpecification clusterSpecification, - String applicationName, + String jobName, String yarnClusterEntrypoint, JobGraph jobGraph, Boolean detached) throws ReflectiveOperationException { @@ -96,12 +106,13 @@ protected final ClusterClientProvider deployInternal( return (ClusterClientProvider) method.invoke( clusterDescriptor, clusterSpecification, - applicationName, + jobName, yarnClusterEntrypoint, jobGraph, detached); } + /** Creates a descriptor for an existing YARN cluster under the requested Hadoop user. */ protected final Tuple2 getYarnClusterDescriptor( Configuration flinkConfig, String user) throws FlinkException { @@ -109,11 +120,13 @@ protected final Tuple2 getYarnClusterDescr user, () -> createYarnClusterDescriptor(flinkConfig)); } + /** Creates a descriptor for an existing YARN cluster under the current Hadoop user. */ protected final Tuple2 getYarnClusterDescriptor( Configuration flinkConfig) throws FlinkException { return getYarnClusterDescriptor(flinkConfig, ""); } + /** Creates the descriptor and resource specification used for YARN cluster deployment. */ protected final Tuple2 getYarnClusterDeployDescriptor( Configuration flinkConfig, String user) throws FlinkException { @@ -121,11 +134,13 @@ protected final Tuple2 getYarnClust user, () -> createYarnClusterDeployDescriptor(flinkConfig)); } + /** Creates a YARN deployment descriptor under the current Hadoop user. */ protected final Tuple2 getYarnClusterDeployDescriptor( Configuration flinkConfig) throws FlinkException { return getYarnClusterDeployDescriptor(flinkConfig, ""); } + /** Logs the resource specification immediately before cluster deployment. */ protected void logClusterSpecification(ClusterSpecification clusterSpecification) { logInfo( String.format( @@ -135,6 +150,7 @@ protected void logClusterSpecification(ClusterSpecification clusterSpecification clusterSpecification)); } + /** Logs the YARN application identity after a successful deployment. */ protected void logYarnJobStarted(Object applicationId) { logInfo( String.format( @@ -144,9 +160,10 @@ protected void logYarnJobStarted(Object applicationId) { applicationId)); } + /** Resolves the configured application ID and creates its YARN descriptor. */ private Tuple2 createYarnClusterDescriptor( Configuration flinkConfig) throws FlinkException { - return callAsFlinkException( + return execute( () -> { YarnClusterClientFactory clientFactory = new YarnClusterClientFactory(); ApplicationId yarnClusterId = clientFactory.getClusterId(flinkConfig); @@ -159,9 +176,10 @@ private Tuple2 createYarnClusterDescriptor }); } + /** Creates the resource specification and descriptor for a new YARN cluster. */ private Tuple2 createYarnClusterDeployDescriptor( Configuration flinkConfig) throws FlinkException { - return callAsFlinkException( + return execute( () -> { YarnClusterClientFactory clientFactory = new YarnClusterClientFactory(); ClusterSpecification clusterSpecification = @@ -172,33 +190,35 @@ private Tuple2 createYarnClusterDep }); } + /** Retrieves the target YARN cluster, executes an action, and closes all client resources. */ private O executeClientAction( - SavepointRequest request, Configuration flinkConf, - ClientAction actionFunc) throws FlinkException { - return callAsFlinkException( + AbstractSavepointRequest request, Configuration flinkConf, + ClientAction action) throws FlinkException { + return execute( () -> { JobID jobID = parseJobId(request.jobId()); - FlinkConfigurationOps.setIfPresent(flinkConf, YarnConfigOptions.APPLICATION_ID, request.clusterId()); + flinkConf.set(YarnConfigOptions.APPLICATION_ID, request.clusterId()); Tuple2 descriptor = getYarnClusterDescriptor(flinkConf); ClusterClient clusterClient = null; try { clusterClient = descriptor._2.retrieve(descriptor._1).getClusterClient(); - return applyClientAction(request, actionFunc, jobID, clusterClient); + return applyClientAction(request, action, jobID, clusterClient); } finally { Utils.close(clusterClient, descriptor._2); } }); } + /** Maps action failures to the stable Flink client exception contract. */ private O applyClientAction( - SavepointRequest request, - ClientAction actionFunc, + AbstractSavepointRequest request, + ClientAction action, JobID jobID, ClusterClient clusterClient) throws FlinkException { - return callAsFlinkExceptionMapping( - () -> actionFunc.apply(jobID, clusterClient), + return executeAndMapError( + () -> action.apply(jobID, clusterClient), e -> new FlinkException( "[StreamPark] Do " + request.getClass().getSimpleName() @@ -210,29 +230,23 @@ private O applyClientAction( e)); } - private T accessYarnClusterDescriptor(String user, YarnDescriptorSupplier func) throws FlinkException { + /** Executes descriptor creation as the requested Hadoop user and adds operation context. */ + private T accessYarnClusterDescriptor( + String user, + YarnDescriptorSupplier operation) throws FlinkException { try { - return doAsYarnClusterDescriptor(user, func); + return doAsYarnClusterDescriptor(user, operation); } catch (FlinkException e) { throw new FlinkException("[StreamPark] access ClusterDescriptor error: " + e.getMessage(), e); } } + /** Lazily resolves and caches Flink's internal YARN deployment method. */ private Method getDeployInternalMethod() throws NoSuchMethodException { if (deployInternalMethod == null) { synchronized (this) { if (deployInternalMethod == null) { - Class[] paramClass = - new Class[]{ - ClusterSpecification.class, - String.class, - String.class, - JobGraph.class, - boolean.class - }; - Method deployInternal = - YarnClusterDescriptor.class.getDeclaredMethod( - "deployInternal", paramClass); + Method deployInternal = getDeployMethod(); deployInternal.setAccessible(true); deployInternalMethod = deployInternal; } @@ -241,7 +255,24 @@ private Method getDeployInternalMethod() throws NoSuchMethodException { return deployInternalMethod; } - private T doAsYarnClusterDescriptor(String user, YarnDescriptorSupplier func) throws FlinkException { + /** Resolves the private method used by Flink to deploy a per-job YARN cluster. */ + @Nonnull + private static Method getDeployMethod() throws NoSuchMethodException { + Class[] paramClass = + new Class[]{ + ClusterSpecification.class, + String.class, + String.class, + JobGraph.class, + boolean.class + }; + return YarnClusterDescriptor.class.getDeclaredMethod("deployInternal", paramClass); + } + + /** Runs descriptor access through Hadoop UGI, creating a proxy user only when required. */ + private T doAsYarnClusterDescriptor( + String user, + YarnDescriptorSupplier operation) throws FlinkException { UserGroupInformation ugi = HadoopUtils.getUgi(); UserGroupInformation finalUgi = user != null @@ -254,7 +285,7 @@ private T doAsYarnClusterDescriptor(String user, YarnDescriptorSupplier f return finalUgi.doAs( (PrivilegedAction) () -> { try { - return func.get(); + return operation.get(); } catch (FlinkException e) { throw new IllegalStateException(e); } catch (Exception e) { diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.java index 0c9b484ff5..1ad6a275bb 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeApplicationClient.java @@ -19,11 +19,10 @@ import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.util.Tuple2; -import org.apache.streampark.flink.client.bean.SubmitRequestResolver; -import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.SavepointRequest; import org.apache.streampark.flink.client.request.SubmitRequest; -import org.apache.streampark.flink.client.request.TriggerSavepointRequest; import org.apache.streampark.flink.client.response.CancelResponse; import org.apache.streampark.flink.client.response.SavepointResponse; import org.apache.streampark.flink.client.response.SubmitResponse; @@ -42,7 +41,7 @@ import com.google.common.collect.Lists; -/** Kubernetes native application mode submit. */ +/** Submits Flink jobs to application-scoped Kubernetes clusters. */ public final class KubernetesNativeApplicationClient extends AbstractKubernetesNativeClient { public static final KubernetesNativeApplicationClient INSTANCE = @@ -51,9 +50,11 @@ public final class KubernetesNativeApplicationClient extends AbstractKubernetesN private KubernetesNativeApplicationClient() { } + /** Deploys an application cluster from the image produced by the build pipeline. */ @Override - protected SubmitResponse doSubmit(SubmitRequest submitRequest, + protected SubmitResponse doSubmit(ResolvedSubmitRequest resolved, Configuration flinkConfig) throws FlinkException { + SubmitRequest submitRequest = resolved.request(); if (StringUtils.isBlank(submitRequest.clusterId())) { throw new IllegalArgumentException( @@ -62,22 +63,21 @@ protected SubmitResponse doSubmit(SubmitRequest submitRequest, flinkConfig.get(DeploymentOptions.TARGET))); } - return callAsFlinkException( + return execute( () -> { - SubmitRequestResolver.validateBuildResult(submitRequest); - DockerImageBuildResponse buildResult = (DockerImageBuildResponse) submitRequest.buildResult(); - FlinkConfigurationOps.setIfPresent( - flinkConfig, + flinkConfig.set( PipelineOptions.JARS, Lists.newArrayList(buildResult.dockerInnerMainJarPath())); - FlinkConfigurationOps.setIfPresent( - flinkConfig, KubernetesConfigOptions.CONTAINER_IMAGE, buildResult.flinkImageTag()); + + flinkConfig.set( + KubernetesConfigOptions.CONTAINER_IMAGE, buildResult.flinkImageTag()); Tuple2 descriptorAndSpec = - getK8sClusterDescriptorAndSpecification(flinkConfig); + createClusterDescriptorAndSpec(flinkConfig); + KubernetesClusterDescriptor clusterDescriptor = descriptorAndSpec._1; ClusterSpecification clusterSpecification = descriptorAndSpec._2; ClusterClient clusterClient = null; @@ -97,24 +97,28 @@ protected SubmitResponse doSubmit(SubmitRequest submitRequest, submitRequest.jobId(), clusterClient.getWebInterfaceURL()); } finally { - closeSubmissionResources(submitRequest, clusterClient, clusterDescriptor); + closeSubmissionResources(clusterClient, clusterDescriptor); } }); } + /** Cancels the job and removes its application-scoped Kubernetes cluster. */ @Override - protected CancelResponse doCancel( - CancelRequest cancelRequest, + protected CancelResponse doCancel(CancelRequest cancelRequest, Configuration flinkConf) throws FlinkException { - setK8sDeployTarget(flinkConf, FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION); + flinkConf.set( + DeploymentOptions.TARGET, + FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION.getName()); return super.doCancel(cancelRequest, flinkConf); } + /** Triggers a savepoint through the application-scoped Kubernetes cluster. */ @Override - protected SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest request, + protected SavepointResponse doTriggerSavepoint(SavepointRequest request, Configuration flinkConf) throws FlinkException { - setK8sDeployTarget(flinkConf, FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION); + flinkConf.set( + DeploymentOptions.TARGET, + FlinkDeployMode.KUBERNETES_NATIVE_APPLICATION.getName()); return super.doTriggerSavepoint(request, flinkConf); } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.java index 3bfb3c5cb7..fbcbb9ab72 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/KubernetesNativeSessionClient.java @@ -20,15 +20,14 @@ import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.util.Tuple2; import org.apache.streampark.common.util.Utils; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; import org.apache.streampark.flink.client.bean.SessionClusterRestClient; -import org.apache.streampark.flink.client.bean.SubmitRequestResolver; -import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.request.AbstractClusterRequest; import org.apache.streampark.flink.client.request.CancelRequest; -import org.apache.streampark.flink.client.request.ClusterRequest; import org.apache.streampark.flink.client.request.DeployRequest; +import org.apache.streampark.flink.client.request.SavepointRequest; import org.apache.streampark.flink.client.request.ShutdownRequest; import org.apache.streampark.flink.client.request.SubmitRequest; -import org.apache.streampark.flink.client.request.TriggerSavepointRequest; import org.apache.streampark.flink.client.response.CancelResponse; import org.apache.streampark.flink.client.response.DeployResponse; import org.apache.streampark.flink.client.response.SavepointResponse; @@ -55,7 +54,7 @@ import java.io.File; -/** Kubernetes native session mode submit. */ +/** Manages Flink jobs and clusters in Kubernetes native session mode. */ public final class KubernetesNativeSessionClient extends AbstractKubernetesNativeClient { public static final KubernetesNativeSessionClient INSTANCE = new KubernetesNativeSessionClient(); @@ -63,54 +62,7 @@ public final class KubernetesNativeSessionClient extends AbstractKubernetesNativ private KubernetesNativeSessionClient() { } - @Override - protected SubmitResponse doSubmit( - SubmitRequest submitRequest, - Configuration flinkConfig) throws FlinkException { - if (StringUtils.isBlank(submitRequest.clusterId())) { - throw new IllegalArgumentException( - String.format( - "[flink-submit] submit flink job failed, clusterId is null, mode=%s", - flinkConfig.get(DeploymentOptions.TARGET))); - } - - return this.restApiSubmit(submitRequest, flinkConfig, SubmitRequestResolver.userJarFile(submitRequest)); - } - - /** Submit flink session job via rest api. */ - private SubmitResponse restApiSubmit( - SubmitRequest submitRequest, - Configuration flinkConfig, - File fatJar) throws FlinkException { - return callAsFlinkException( - () -> { - ClusterKey clusterKey = - ClusterKey.builder() - .executeMode(FlinkKubernetesDeployMode.SESSION) - .namespace(submitRequest.kubernetesNamespace()) - .clusterId(submitRequest.clusterId()) - .build(); - String jmRestUrl = - KubernetesRetriever.retrieveFlinkRestUrl(clusterKey) - .orElseThrow( - () -> new FlinkException( - "[flink-submit] retrieve flink session rest url failed, clusterKey=" - + clusterKey)); - String jobId = - SessionClusterRestClient.submit(jmRestUrl, fatJar, flinkConfig); - return new SubmitResponse( - clusterKey.clusterId(), flinkConfig.toMap(), jobId, jmRestUrl); - }); - } - - @Override - protected CancelResponse doCancel( - CancelRequest cancelRequest, - Configuration flinkConfig) throws FlinkException { - setK8sDeployTarget(flinkConfig, FlinkDeployMode.KUBERNETES_NATIVE_SESSION); - return super.doCancel(cancelRequest, flinkConfig); - } - + /** Deploys or reconnects to the requested Kubernetes session cluster. */ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { logInfo( String.format( @@ -136,16 +88,16 @@ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { deployRequest.kubernetesDeploySpec().flinkImage(), deployRequest.properties())); - Configuration flinkConfig = getFlinkK8sConfig(deployRequest); - FlinkKubeClient kubeClient = - FlinkKubeClientFactory.getInstance().fromConfiguration(flinkConfig, "client"); + Configuration flinkConfig = getFlinkKubernetesConfig(deployRequest); + + FlinkKubeClient kubeClient = FlinkKubeClientFactory.getInstance().fromConfiguration(flinkConfig, "client"); KubernetesClusterDescriptor clusterDescriptor = null; ClusterClient client = null; try { Tuple2 kubernetesClusterDescriptor = - getK8sClusterDescriptorAndSpecification(flinkConfig); + createClusterDescriptorAndSpec(flinkConfig); clusterDescriptor = kubernetesClusterDescriptor._1; FlinkKubernetesClient kubeClientWrapper = new FlinkKubernetesClient(kubeClient); @@ -167,22 +119,23 @@ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { } } + /** Stops a Kubernetes session cluster when it exists. */ public ShutdownResponse shutdown(ShutdownRequest shutdownRequest) throws Exception { FlinkKubeClient kubeClient = null; try { - Configuration flinkConfig = getFlinkK8sConfig(shutdownRequest); + Configuration flinkConfig = getFlinkKubernetesConfig(shutdownRequest); kubeClient = FlinkKubeClientFactory.getInstance().fromConfiguration(flinkConfig, "client"); FlinkKubernetesClient kubeClientWrapper = new FlinkKubernetesClient(kubeClient); - boolean stopAndCleanupState = + boolean clusterExists = shutdownRequest.clusterId() != null && kubeClientWrapper.getService(shutdownRequest.clusterId()).isPresent(); - if (stopAndCleanupState) { + if (clusterExists) { kubeClient.stopAndCleanupCluster(shutdownRequest.clusterId()); - return new ShutdownResponse(shutdownRequest.clusterId()); } - return null; + // Shutdown is idempotent: an already absent cluster is the desired final state. + return new ShutdownResponse(shutdownRequest.clusterId()); } catch (Exception e) { logError( "shutdown flink session fail in " + shutdownRequest.deployMode() + " mode", e); @@ -192,57 +145,111 @@ public ShutdownResponse shutdown(ShutdownRequest shutdownRequest) throws Excepti } } + /** Submits a job to an existing Kubernetes session cluster through its REST endpoint. */ + @Override + protected SubmitResponse doSubmit( + ResolvedSubmitRequest resolved, + Configuration flinkConfig) throws FlinkException { + SubmitRequest submitRequest = resolved.request(); + if (StringUtils.isBlank(submitRequest.clusterId())) { + throw new IllegalArgumentException( + String.format( + "[flink-submit] submit flink job failed, clusterId is null, mode=%s", + flinkConfig.get(DeploymentOptions.TARGET))); + } + + return restApiSubmit(submitRequest, flinkConfig, resolved.getUserJarFile()); + } + + /** Cancels a job while retaining its Kubernetes session cluster. */ + @Override + protected CancelResponse doCancel( + CancelRequest cancelRequest, + Configuration flinkConfig) throws FlinkException { + flinkConfig.set( + DeploymentOptions.TARGET, + FlinkDeployMode.KUBERNETES_NATIVE_SESSION.getName()); + return super.doCancel(cancelRequest, flinkConfig); + } + + /** Triggers a savepoint against an existing Kubernetes session cluster. */ @Override protected SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest triggerSavepointRequest, + SavepointRequest triggerSavepointRequest, Configuration flinkConfig) throws FlinkException { - FlinkConfigurationOps.setIfPresent( - flinkConfig, + flinkConfig.set( DeploymentOptions.TARGET, FlinkDeployMode.KUBERNETES_NATIVE_SESSION.getName()); return super.doTriggerSavepoint(triggerSavepointRequest, flinkConfig); } - private Configuration getFlinkK8sConfig(ClusterRequest deployRequest) throws Exception { + /** Submits the uploaded job JAR and returns the session cluster job identity. */ + private SubmitResponse restApiSubmit( + SubmitRequest submitRequest, + Configuration flinkConfig, + File fatJar) throws FlinkException { + return execute( + () -> { + ClusterKey clusterKey = + ClusterKey.builder() + .executeMode(FlinkKubernetesDeployMode.SESSION) + .namespace(submitRequest.kubernetesNamespace()) + .clusterId(submitRequest.clusterId()) + .build(); + String jmRestUrl = + KubernetesRetriever.retrieveFlinkRestUrl(clusterKey) + .orElseThrow( + () -> new FlinkException( + "[flink-submit] retrieve flink session rest url failed, clusterKey=" + + clusterKey)); + String jobId = + SessionClusterRestClient.submit(jmRestUrl, fatJar, flinkConfig); + return new SubmitResponse( + clusterKey.clusterId(), flinkConfig.toMap(), jobId, jmRestUrl); + }); + } + + /** Builds the Flink configuration shared by session deployment and shutdown operations. */ + private Configuration getFlinkKubernetesConfig(AbstractClusterRequest request) throws Exception { Configuration flinkConfig = extractConfiguration( - deployRequest.flinkVersion().getFlinkHome(), deployRequest.properties()); + request.flinkVersion().getFlinkHome(), request.properties()); - FlinkConfigurationOps.setIfPresent(flinkConfig, DeploymentOptions.TARGET, - KubernetesDeploymentTarget.SESSION.getName()); + flinkConfig.set( + DeploymentOptions.TARGET, KubernetesDeploymentTarget.SESSION.getName()); - FlinkConfigurationOps.setIfPresent( - flinkConfig, - KubernetesConfigOptions.NAMESPACE, - deployRequest.kubernetesDeploySpec().kubernetesNamespace()); + if (StringUtils.isNotEmpty( + request.kubernetesDeploySpec().kubernetesNamespace())) { + flinkConfig.set( + KubernetesConfigOptions.NAMESPACE, + request.kubernetesDeploySpec().kubernetesNamespace()); + } - FlinkConfigurationOps.setIfPresent( - flinkConfig, - KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT, - deployRequest.kubernetesDeploySpec().serviceAccount()); + if (StringUtils.isNotEmpty(request.kubernetesDeploySpec().serviceAccount())) { + flinkConfig.set( + KubernetesConfigOptions.KUBERNETES_SERVICE_ACCOUNT, + request.kubernetesDeploySpec().serviceAccount()); + } - FlinkConfigurationOps.setIfPresent( - flinkConfig, KubernetesConfigOptions.CLUSTER_ID, deployRequest.clusterId()); + flinkConfig.set(KubernetesConfigOptions.CLUSTER_ID, request.clusterId()); - FlinkConfigurationOps.setIfPresent( - flinkConfig, - KubernetesConfigOptions.CONTAINER_IMAGE, - deployRequest.kubernetesDeploySpec().flinkImage()); + if (StringUtils.isNotEmpty(request.kubernetesDeploySpec().flinkImage())) { + flinkConfig.set( + KubernetesConfigOptions.CONTAINER_IMAGE, + request.kubernetesDeploySpec().flinkImage()); + } - FlinkConfigurationOps.setIfPresent( - flinkConfig, + flinkConfig.set( KubernetesConfigOptions.REST_SERVICE_EXPOSED_TYPE, ServiceExposedType.valueOf( - deployRequest.kubernetesDeploySpec().flinkRestExposedType().getName())); + request.kubernetesDeploySpec().flinkRestExposedType().getName())); - FlinkConfigurationOps.setIfPresent( - flinkConfig, + flinkConfig.set( KubernetesConfigOptions.KUBE_CONFIG_FILE, - getDefaultKubernetesConf(deployRequest.kubernetesDeploySpec().kubeConf())); + getDefaultKubernetesConf(request.kubernetesDeploySpec().kubeConf())); - FlinkConfigurationOps.setIfPresent( - flinkConfig, + flinkConfig.set( DeploymentOptionsInternal.CONF_DIR, - deployRequest.flinkVersion().getFlinkHome() + "/conf"); + request.flinkVersion().getFlinkHome() + "/conf"); return flinkConfig; } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/LocalClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/LocalClient.java index 5be4d82b65..a6eb60cbe1 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/LocalClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/LocalClient.java @@ -18,11 +18,10 @@ package org.apache.streampark.flink.client.impl; import org.apache.streampark.flink.client.bean.FlinkJobGraphBuilder; -import org.apache.streampark.flink.client.bean.SubmitRequestResolver; -import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; import org.apache.streampark.flink.client.request.CancelRequest; +import org.apache.streampark.flink.client.request.SavepointRequest; import org.apache.streampark.flink.client.request.SubmitRequest; -import org.apache.streampark.flink.client.request.TriggerSavepointRequest; import org.apache.streampark.flink.client.response.CancelResponse; import org.apache.streampark.flink.client.response.SavepointResponse; import org.apache.streampark.flink.client.response.SubmitResponse; @@ -40,7 +39,7 @@ import org.apache.flink.runtime.minicluster.MiniClusterConfiguration; import org.apache.flink.util.FlinkException; -/** Submit Flink jobs to a local mini cluster. */ +/** Submits and manages Flink jobs in an embedded local cluster. */ public final class LocalClient extends AbstractFlinkClient { public static final LocalClient INSTANCE = new LocalClient(); @@ -48,25 +47,25 @@ public final class LocalClient extends AbstractFlinkClient { private LocalClient() { } + /** Adds the resolved job name used by the embedded cluster. */ @Override - protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - FlinkConfigurationOps.setIfPresent( - flinkConfig, - PipelineOptions.NAME, - SubmitRequestResolver.effectiveApplicationName(submitRequest)); + protected void setConfig(ResolvedSubmitRequest resolved, Configuration flinkConfig) { + flinkConfig.set(PipelineOptions.NAME, resolved.getJobName()); logEffectiveSubmitConfiguration(flinkConfig); } + /** Creates an embedded cluster and submits the request's JobGraph. */ @Override protected SubmitResponse doSubmit( - SubmitRequest submitRequest, + ResolvedSubmitRequest resolved, Configuration flinkConfig) throws FlinkException { - return callAsFlinkException( + SubmitRequest submitRequest = resolved.request(); + return execute( () -> { FlinkJobGraphBuilder.Result job = buildJobGraph( flinkConfig, - submitRequest, - SubmitRequestResolver.userJarFile(submitRequest)); + resolved, + resolved.getUserJarFile()); MiniClusterClient client = null; try { client = createLocalCluster(flinkConfig); @@ -74,18 +73,20 @@ protected SubmitResponse doSubmit( return new SubmitResponse( jobId, flinkConfig.toMap(), jobId, client.getWebInterfaceURL()); } finally { - closeSubmissionResources(submitRequest, job.program(), client); + closeSubmissionResources(job.program(), client); } }); } + /** Delegates savepoint access to the standalone cluster protocol. */ @Override protected SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest savepointRequest, + SavepointRequest savepointRequest, Configuration flinkConfig) throws FlinkException { return RemoteClient.INSTANCE.doTriggerSavepoint(savepointRequest, flinkConfig); } + /** Delegates cancellation to the standalone cluster protocol. */ @Override protected CancelResponse doCancel( CancelRequest cancelRequest, @@ -93,8 +94,9 @@ protected CancelResponse doCancel( return RemoteClient.INSTANCE.doCancel(cancelRequest, flinkConfig); } + /** Creates and starts the mini cluster used for one local submission. */ private MiniClusterClient createLocalCluster(Configuration flinkConfig) throws Exception { - FlinkConfigurationOps.setIfPresent(flinkConfig, JobManagerOptions.PORT, 0); + flinkConfig.set(JobManagerOptions.PORT, 0); int numTaskManagers = flinkConfig.getInteger( @@ -115,11 +117,11 @@ private MiniClusterClient createLocalCluster(Configuration flinkConfig) throws E String host = "localhost"; int port = cluster.getRestAddress().get().getPort(); - FlinkConfigurationOps.setIfPresent(flinkConfig, JobManagerOptions.ADDRESS, host); - FlinkConfigurationOps.setIfPresent(flinkConfig, JobManagerOptions.PORT, port); - FlinkConfigurationOps.setIfPresent(flinkConfig, RestOptions.ADDRESS, host); - FlinkConfigurationOps.setIfPresent(flinkConfig, RestOptions.PORT, port); - FlinkConfigurationOps.setIfPresent(flinkConfig, DeploymentOptions.TARGET, RemoteExecutor.NAME); + flinkConfig.set(JobManagerOptions.ADDRESS, host); + flinkConfig.set(JobManagerOptions.PORT, port); + flinkConfig.set(RestOptions.ADDRESS, host); + flinkConfig.set(RestOptions.PORT, port); + flinkConfig.set(DeploymentOptions.TARGET, RemoteExecutor.NAME); logInfo(String.format("%nStarting local Flink cluster (host: localhost, port: %d).%n", port)); diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/RemoteClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/RemoteClient.java index 5c3e6dbd48..0795503d9d 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/RemoteClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/RemoteClient.java @@ -17,15 +17,15 @@ package org.apache.streampark.flink.client.impl; +import org.apache.streampark.common.util.AssertUtils; import org.apache.streampark.common.util.ClassLoaderUtils; import org.apache.streampark.common.util.Tuple2; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; import org.apache.streampark.flink.client.bean.SessionClusterRestClient; -import org.apache.streampark.flink.client.bean.SubmitRequestResolver; -import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.request.AbstractSavepointRequest; import org.apache.streampark.flink.client.request.CancelRequest; import org.apache.streampark.flink.client.request.SavepointRequest; import org.apache.streampark.flink.client.request.SubmitRequest; -import org.apache.streampark.flink.client.request.TriggerSavepointRequest; import org.apache.streampark.flink.client.response.CancelResponse; import org.apache.streampark.flink.client.response.SavepointResponse; import org.apache.streampark.flink.client.response.SubmitResponse; @@ -42,8 +42,9 @@ import org.apache.flink.util.FlinkException; import java.io.File; +import java.util.Map; -/** Submit Flink jobs to a remote standalone cluster. */ +/** Submits and manages Flink jobs on a remote standalone cluster. */ public final class RemoteClient extends AbstractFlinkClient { public static final RemoteClient INSTANCE = new RemoteClient(); @@ -51,24 +52,27 @@ public final class RemoteClient extends AbstractFlinkClient { private RemoteClient() { } + /** Leaves standalone-specific configuration to Flink's standard CLI processing. */ @Override - protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - // no extra configuration + protected void setConfig(ResolvedSubmitRequest resolved, Configuration flinkConfig) { + // Standalone submission requires no deployment-mode-specific options. } + /** Tries native JobGraph submission before using the standalone REST endpoint. */ @Override protected SubmitResponse doSubmit( - SubmitRequest submitRequest, + ResolvedSubmitRequest resolved, Configuration flinkConfig) throws FlinkException { - return callAsFlinkException( + return execute( () -> trySubmit( - submitRequest, + resolved, flinkConfig, - SubmitRequestResolver.userJarFile(submitRequest), + resolved.getUserJarFile(), this::jobGraphSubmit, this::restApiSubmit)); } + /** Cancels a job through the remote standalone cluster client. */ @Override protected CancelResponse doCancel( CancelRequest cancelRequest, @@ -79,9 +83,10 @@ protected CancelResponse doCancel( (jobId, clusterClient) -> toCancelResponse(cancelRequest, jobId, clusterClient)); } + /** Triggers a savepoint through the remote standalone cluster client. */ @Override protected SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest savepointRequest, + SavepointRequest savepointRequest, Configuration flinkConfig) throws FlinkException { return executeClientAction( savepointRequest, @@ -89,20 +94,21 @@ protected SavepointResponse doTriggerSavepoint( (jobId, clusterClient) -> toSavepointResponse(savepointRequest, jobId, clusterClient)); } - /** Submit flink session job via rest api. */ + /** Submits a job through the standalone cluster's REST API. */ private SubmitResponse restApiSubmit( - SubmitRequest submitRequest, + ResolvedSubmitRequest resolved, Configuration flinkConfig, File fatJar) throws FlinkException { - return callAsFlinkException( + SubmitRequest submitRequest = resolved.request(); + return execute( () -> { - Tuple2 standAloneDescriptor = - getStandAloneClusterDescriptor(flinkConfig); - StandaloneClusterId yarnClusterId = standAloneDescriptor._1; - StandaloneClusterDescriptor clusterDescriptor = standAloneDescriptor._2; + Tuple2 standaloneDescriptor = + standaloneClusterDescriptor(flinkConfig); + StandaloneClusterId clusterId = standaloneDescriptor._1; + StandaloneClusterDescriptor clusterDescriptor = standaloneDescriptor._2; ClusterClient client = null; try { - client = clusterDescriptor.retrieve(yarnClusterId).getClusterClient(); + client = clusterDescriptor.retrieve(clusterId).getClusterClient(); String jobId = SessionClusterRestClient.submit( client.getWebInterfaceURL(), fatJar, flinkConfig); @@ -113,54 +119,52 @@ private SubmitResponse restApiSubmit( return new SubmitResponse( null, flinkConfig.toMap(), jobId, client.getWebInterfaceURL()); } finally { - closeSubmissionResources(submitRequest, client, clusterDescriptor); + closeSubmissionResources(client, clusterDescriptor); } }); } - /** Submit flink session job with building JobGraph via Standalone ClusterClient api. */ + /** Submits a JobGraph through the standalone cluster client. */ private SubmitResponse jobGraphSubmit( - SubmitRequest submitRequest, + ResolvedSubmitRequest resolved, Configuration flinkConfig, File jarFile) throws FlinkException { - Tuple2 standAloneDescriptor = - getStandAloneClusterDescriptor(flinkConfig); + Tuple2 standaloneDescriptor = + standaloneClusterDescriptor(flinkConfig); return submitJobGraphToCluster( - submitRequest, + resolved, flinkConfig, jarFile, - () -> standAloneDescriptor._2.retrieve(standAloneDescriptor._1).getClusterClient(), + () -> standaloneDescriptor._2.retrieve(standaloneDescriptor._1).getClusterClient(), () -> null, - standAloneDescriptor._2); + standaloneDescriptor._2); } - private O executeClientAction( - R request, - Configuration flinkConfig, - ClientAction actFunc) throws FlinkException { - return callAsFlinkException( + /** Retrieves a standalone cluster, executes a job action, and closes its client resources. */ + private O executeClientAction( + R request, + Configuration flinkConfig, + ClientAction action) throws FlinkException { + return execute( () -> { - FlinkConfigurationOps.setIfPresent( - flinkConfig, DeploymentOptions.TARGET, request.deployMode().getName()); - FlinkConfigurationOps.setIfPresent( - flinkConfig, - RestOptions.ADDRESS, - request.properties().get(RestOptions.ADDRESS.key()).toString()); - FlinkConfigurationOps.setIfPresent( - flinkConfig, - RestOptions.PORT, - Integer.parseInt( - request.properties().get(RestOptions.PORT.key()).toString())); + Map properties = request.properties(); + Object address = properties.get(RestOptions.ADDRESS.key()); + Object port = properties.get(RestOptions.PORT.key()); + AssertUtils.required(address != null, "Remote Flink REST address is not configured"); + AssertUtils.required(port != null, "Remote Flink REST port is not configured"); + + flinkConfig.set(DeploymentOptions.TARGET, request.deployMode().getName()); + flinkConfig.set(RestOptions.ADDRESS, address.toString()); + flinkConfig.set(RestOptions.PORT, Integer.parseInt(port.toString())); logEffectiveSubmitConfiguration(flinkConfig); Tuple2 descriptor = - getStandAloneClusterDescriptor(flinkConfig); - ClusterClient clusterClient = - descriptor._2.retrieve(descriptor._1).getClusterClient(); + standaloneClusterDescriptor(flinkConfig); + ClusterClient clusterClient = null; try { - return actFunc.apply(JobID.fromHexString(request.jobId()), clusterClient); + clusterClient = descriptor._2.retrieve(descriptor._1).getClusterClient(); + return action.apply(JobID.fromHexString(request.jobId()), clusterClient); } finally { - clusterClient.close(); - descriptor._2.close(); + closeSubmissionResources(clusterClient, descriptor._2); } }, e -> { @@ -169,8 +173,9 @@ private O executeClientAction( }); } - private Tuple2 getStandAloneClusterDescriptor( - Configuration flinkConfig) { + /** Creates the standalone descriptor using the client module's own Flink service loader. */ + private Tuple2 standaloneClusterDescriptor( + Configuration flinkConfig) { // DefaultClusterClientServiceLoader is bound to the Flink version bundled with this module // (loaded by this class's own classloader), but the calling thread's context classloader may // currently be a target-version shims classloader (see FlinkShimsProxy). Its internal diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java index c24f5f2e06..4a08f13adc 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnApplicationClient.java @@ -26,8 +26,7 @@ import org.apache.streampark.common.util.HdfsUtils; import org.apache.streampark.common.util.Tuple2; import org.apache.streampark.flink.client.bean.RemoteWorkspace; -import org.apache.streampark.flink.client.bean.SubmitRequestResolver; -import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; import org.apache.streampark.flink.client.request.SubmitRequest; import org.apache.streampark.flink.client.response.SubmitResponse; import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse; @@ -49,7 +48,7 @@ import java.util.Collections; import java.util.List; -/** Yarn application mode submit. */ +/** Submits Flink jobs to application-scoped YARN clusters. */ public final class YarnApplicationClient extends AbstractYarnClient { public static final YarnApplicationClient INSTANCE = new YarnApplicationClient(); @@ -59,9 +58,11 @@ public final class YarnApplicationClient extends AbstractYarnClient { private YarnApplicationClient() { } + /** Configures remote artifacts and PyFlink resources for YARN application mode. */ @Override - protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - super.setConfig(submitRequest, flinkConfig); + protected void setConfig(ResolvedSubmitRequest resolved, Configuration flinkConfig) { + SubmitRequest submitRequest = resolved.request(); + super.setConfig(resolved, flinkConfig); List providedLibs = new ArrayList<>(); RemoteWorkspace hdfsWorkspace = RemoteWorkspace.resolve(submitRequest.flinkVersion()); @@ -82,68 +83,58 @@ protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) } } - FlinkConfigurationOps.setIfPresent(flinkConfig, YarnConfigOptions.PROVIDED_LIB_DIRS, providedLibs); - FlinkConfigurationOps.setIfPresent( - flinkConfig, YarnConfigOptions.FLINK_DIST_JAR, hdfsWorkspace.flinkDistJar()); - FlinkConfigurationOps.setIfPresent( - flinkConfig, + flinkConfig.set(YarnConfigOptions.PROVIDED_LIB_DIRS, providedLibs); + flinkConfig.set(YarnConfigOptions.FLINK_DIST_JAR, hdfsWorkspace.flinkDistJar()); + flinkConfig.set( PipelineOptions.JARS, Collections.singletonList( ((ShadedBuildResponse) submitRequest.buildResult()).shadedJarPath())); - FlinkConfigurationOps.setIfPresent( - flinkConfig, + flinkConfig.set( YarnConfigOptions.APPLICATION_NAME, - SubmitRequestResolver.effectiveApplicationName(submitRequest)); - FlinkConfigurationOps.setIfPresent( - flinkConfig, + resolved.getJobName()); + flinkConfig.set( YarnConfigOptions.APPLICATION_TYPE, submitRequest.applicationType().getName()); if (submitRequest.jobType() == FlinkJobType.PYFLINK) { - File userJar = SubmitRequestResolver.userJarFile(submitRequest); - AssertUtils.required(userJar != null, "PyFlink application archive is missing"); + File userJar = resolved.getUserJarFile(); + AssertUtils.required(userJar != null, "PyFlink job archive is missing"); String pyVenv = WORKSPACE.pythonVenv; AssertUtils.required(FsOperator.hdfs().exists(pyVenv), pyVenv + " File does not exist"); - String localLib = - Workspace.LOCAL.workspace + "/" + submitRequest.id() + "/lib"; + String localLib = Workspace.LOCAL.workspace + "/" + submitRequest.id() + "/lib"; if (FileUtils.exists(localLib) && FileUtils.directoryNotBlank(localLib)) { - FlinkConfigurationOps.setIfPresent(flinkConfig, PipelineOptions.JARS, Arrays.asList(localLib)); + flinkConfig.set(PipelineOptions.JARS, Arrays.asList(localLib)); } ArrayList shipFiles = new ArrayList<>(); shipFiles.add(userJar.getParentFile().getAbsolutePath()); - FlinkConfigurationOps.setIfPresent(flinkConfig, YarnConfigOptions.SHIP_FILES, shipFiles); - FlinkConfigurationOps.setIfPresent( - flinkConfig, - PythonOptions.PYTHON_FILES, - userJar.getParentFile().getName()); - FlinkConfigurationOps.setIfPresent(flinkConfig, PythonOptions.PYTHON_ARCHIVES, pyVenv); - FlinkConfigurationOps.setIfPresent( - flinkConfig, PythonOptions.PYTHON_CLIENT_EXECUTABLE, Constants.PYTHON_EXECUTABLE); - FlinkConfigurationOps.setIfPresent( - flinkConfig, PythonOptions.PYTHON_EXECUTABLE, Constants.PYTHON_EXECUTABLE); + flinkConfig.set(YarnConfigOptions.SHIP_FILES, shipFiles); + flinkConfig.set(PythonOptions.PYTHON_FILES, userJar.getParentFile().getName()); + flinkConfig.set(PythonOptions.PYTHON_ARCHIVES, pyVenv); + flinkConfig.set(PythonOptions.PYTHON_CLIENT_EXECUTABLE, Constants.PYTHON_EXECUTABLE); + flinkConfig.set(PythonOptions.PYTHON_EXECUTABLE, Constants.PYTHON_EXECUTABLE); List args = flinkConfig.get(ApplicationConfiguration.APPLICATION_ARGS); ArrayList argsList = new ArrayList<>(args); argsList.add("-pym"); argsList.add( - userJar - .getName() + userJar.getName() .substring(0, userJar.getName().length() - Constants.PYTHON_SUFFIX.length())); - FlinkConfigurationOps.setIfPresent( - flinkConfig, ApplicationConfiguration.APPLICATION_ARGS, argsList); + flinkConfig.set(ApplicationConfiguration.APPLICATION_ARGS, argsList); } logEffectiveSubmitConfiguration(flinkConfig); } + /** Deploys an application cluster and returns its YARN and REST identities. */ @Override protected SubmitResponse doSubmit( - SubmitRequest submitRequest, + ResolvedSubmitRequest resolved, Configuration flinkConfig) throws FlinkException { - return callAsFlinkException( + SubmitRequest submitRequest = resolved.request(); + return execute( () -> { Tuple2 deployDescriptor = getYarnClusterDeployDescriptor(flinkConfig, submitRequest.hadoopUser()); @@ -168,8 +159,7 @@ protected SubmitResponse doSubmit( "", clusterClient.getWebInterfaceURL()); } finally { - closeSubmissionResources( - submitRequest, clusterClient, clusterDescriptor); + closeSubmissionResources(clusterClient, clusterDescriptor); } }); } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnPerJobClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnPerJobClient.java index 43b077f674..140b23dbb3 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnPerJobClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnPerJobClient.java @@ -20,8 +20,7 @@ import org.apache.streampark.common.util.Tuple2; import org.apache.streampark.common.util.Utils; import org.apache.streampark.flink.client.bean.FlinkJobGraphBuilder; -import org.apache.streampark.flink.client.bean.SubmitRequestResolver; -import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; import org.apache.streampark.flink.client.request.CancelRequest; import org.apache.streampark.flink.client.request.SubmitRequest; import org.apache.streampark.flink.client.response.CancelResponse; @@ -40,7 +39,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ApplicationId; -/** Yarn per-job mode submit. */ +/** Submits and manages Flink jobs in the legacy YARN per-job deployment mode. */ public final class YarnPerJobClient extends AbstractYarnClient { public static final YarnPerJobClient INSTANCE = new YarnPerJobClient(); @@ -48,23 +47,25 @@ public final class YarnPerJobClient extends AbstractYarnClient { private YarnPerJobClient() { } + /** Configures attached per-job deployment and cluster shutdown semantics. */ @Override - protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - super.setConfig(submitRequest, flinkConfig); - FlinkConfigurationOps.setIfPresent( - flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.PER_JOB.getName()); - FlinkConfigurationOps.setIfPresent(flinkConfig, DeploymentOptions.ATTACHED, true); - FlinkConfigurationOps.setIfPresent(flinkConfig, DeploymentOptions.SHUTDOWN_IF_ATTACHED, true); + protected void setConfig(ResolvedSubmitRequest resolved, Configuration flinkConfig) { + super.setConfig(resolved, flinkConfig); + flinkConfig.set(DeploymentOptions.TARGET, YarnDeploymentTarget.PER_JOB.getName()); + flinkConfig.set(DeploymentOptions.ATTACHED, true); + flinkConfig.set(DeploymentOptions.SHUTDOWN_IF_ATTACHED, true); logEffectiveSubmitConfiguration(flinkConfig); } + /** Deploys one YARN cluster around the request's JobGraph. */ @Override protected SubmitResponse doSubmit( - SubmitRequest submitRequest, + ResolvedSubmitRequest resolved, Configuration flinkConfig) throws FlinkException { + SubmitRequest submitRequest = resolved.request(); - return callAsFlinkException( + return execute( () -> { String flinkHome = submitRequest.flinkVersion().getFlinkHome(); @@ -84,14 +85,14 @@ protected SubmitResponse doSubmit( job = buildJobGraph( flinkConfig, - submitRequest, - SubmitRequestResolver.userJarFile(submitRequest)); + resolved, + resolved.getUserJarFile()); JobGraph jobGraph = job.jobGraph(); clusterClient = deployInternal( clusterDescriptor, clusterSpecification, - SubmitRequestResolver.effectiveApplicationName(submitRequest), + resolved.getJobName(), YarnJobClusterEntrypoint.class.getName(), jobGraph, true) @@ -105,7 +106,6 @@ protected SubmitResponse doSubmit( clusterClient.getWebInterfaceURL()); } finally { closeSubmissionResources( - submitRequest, job == null ? null : job.program(), clusterClient, clusterDescriptor); @@ -113,11 +113,12 @@ protected SubmitResponse doSubmit( }); } + /** Cancels the job and then terminates its dedicated YARN cluster. */ @Override protected CancelResponse doCancel( CancelRequest cancelRequest, Configuration flinkConfig) throws FlinkException { - return callAsFlinkException( + return execute( () -> { CancelResponse response = super.doCancel(cancelRequest, flinkConfig); Tuple2 yarnClusterDescriptor = diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java index 6f65848847..a8709c5625 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/main/java/org/apache/streampark/flink/client/impl/YarnSessionClient.java @@ -21,13 +21,12 @@ import org.apache.streampark.common.util.Utils; import org.apache.streampark.flink.client.bean.FlinkJobGraphBuilder; import org.apache.streampark.flink.client.bean.RemoteWorkspace; -import org.apache.streampark.flink.client.bean.SubmitRequestResolver; -import org.apache.streampark.flink.client.configuration.FlinkConfigurationOps; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; import org.apache.streampark.flink.client.request.CancelRequest; import org.apache.streampark.flink.client.request.DeployRequest; +import org.apache.streampark.flink.client.request.SavepointRequest; import org.apache.streampark.flink.client.request.ShutdownRequest; import org.apache.streampark.flink.client.request.SubmitRequest; -import org.apache.streampark.flink.client.request.TriggerSavepointRequest; import org.apache.streampark.flink.client.response.CancelResponse; import org.apache.streampark.flink.client.response.DeployResponse; import org.apache.streampark.flink.client.response.SavepointResponse; @@ -47,12 +46,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; -import org.apache.hadoop.yarn.util.ConverterUtils; import java.util.ArrayList; import java.util.Map; -/** Submit Flink jobs to a YARN session cluster. */ +/** Manages Flink jobs and clusters in YARN session mode. */ public final class YarnSessionClient extends AbstractYarnClient { public static final YarnSessionClient INSTANCE = new YarnSessionClient(); @@ -60,93 +58,11 @@ public final class YarnSessionClient extends AbstractYarnClient { private YarnSessionClient() { } - @Override - protected void setConfig(SubmitRequest submitRequest, Configuration flinkConfig) { - super.setConfig(submitRequest, flinkConfig); - FlinkConfigurationOps.setIfPresent( - flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); - logEffectiveSubmitConfiguration(flinkConfig); - } - - private void deployClusterConfig(DeployRequest deployRequest, Configuration flinkConfig) { - ArrayList shipFiles = new ArrayList<>(); - shipFiles.add(deployRequest.flinkVersion().getFlinkHome() + "/lib"); - shipFiles.add(deployRequest.flinkVersion().getFlinkHome() + "/plugins"); - - RemoteWorkspace hdfsWorkspace = RemoteWorkspace.resolve(deployRequest.flinkVersion()); - FlinkConfigurationOps.setIfPresent( - flinkConfig, YarnConfigOptions.FLINK_DIST_JAR, hdfsWorkspace.flinkDistJar()); - FlinkConfigurationOps.setIfPresent(flinkConfig, YarnConfigOptions.SHIP_FILES, shipFiles); - FlinkConfigurationOps.setIfPresent( - flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); - FlinkConfigurationOps.setIfPresent( - flinkConfig, - DeploymentOptionsInternal.CONF_DIR, - deployRequest.flinkVersion().getFlinkHome() + "/conf"); - - logEffectiveSubmitConfiguration(flinkConfig); - } - - @Override - protected SubmitResponse doSubmit( - SubmitRequest submitRequest, - Configuration flinkConfig) throws FlinkException { - return callAsFlinkException( - () -> { - Tuple2 yarnClusterDescriptor = - getYarnClusterDescriptor(flinkConfig); - ApplicationId yarnClusterId = yarnClusterDescriptor._1; - YarnClusterDescriptor clusterDescriptor = yarnClusterDescriptor._2; - - FlinkJobGraphBuilder.Result job = null; - ClusterClient client = null; - try { - job = - buildJobGraph( - flinkConfig, - submitRequest, - SubmitRequestResolver.userJarFile(submitRequest)); - - client = clusterDescriptor.retrieve(yarnClusterId).getClusterClient(); - String jobId = client.submitJob(job.jobGraph()).get().toString(); - logInfo("Flink job started: jobId=" + jobId + ", applicationId=" + yarnClusterId); - return new SubmitResponse( - yarnClusterId.toString(), - flinkConfig.toMap(), - jobId, - client.getWebInterfaceURL()); - } finally { - closeSubmissionResources( - submitRequest, - job == null ? null : job.program(), - client, - clusterDescriptor); - } - }); - } - - @Override - protected CancelResponse doCancel( - CancelRequest cancelRequest, - Configuration flinkConfig) throws FlinkException { - FlinkConfigurationOps.setIfPresent( - flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); - return super.doCancel(cancelRequest, flinkConfig); - } - - @Override - protected SavepointResponse doTriggerSavepoint( - TriggerSavepointRequest request, - Configuration flinkConfig) throws FlinkException { - FlinkConfigurationOps.setIfPresent( - flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); - return super.doTriggerSavepoint(request, flinkConfig); - } - + /** Deploys a new YARN session cluster or reconnects to a live requested cluster. */ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { logInfo( String.format( - "%n--------------------------------------- flink yarn sesion start " + "%n--------------------------------------- Flink YARN session start " + "---------------------------------------%n" + " userFlinkHome : %s%n" + " flinkVersion : %s%n" @@ -199,66 +115,34 @@ public DeployResponse deploy(DeployRequest deployRequest) throws Exception { } } - private DeployResponse tryReuseExistingYarnSession( - DeployRequest deployRequest, - YarnClusterDescriptor clusterDescriptor) throws Exception { - try { - FinalApplicationStatus applicationStatus = - clusterDescriptor - .getYarnClient() - .getApplicationReport( - ApplicationId.fromString(deployRequest.clusterId())) - .getFinalApplicationStatus(); - if (FinalApplicationStatus.UNDEFINED != applicationStatus) { - return null; - } - ClusterClient yarnClient = - clusterDescriptor - .retrieve(ApplicationId.fromString(deployRequest.clusterId())) - .getClusterClient(); - if (yarnClient.getWebInterfaceURL() == null) { - return null; - } - return new DeployResponse( - yarnClient.getWebInterfaceURL(), - yarnClient.getClusterId().toString(), - null); - } catch (ApplicationNotFoundException e) { - logInfo("this applicationId have not managed by yarn ,need deploy ..."); - return null; - } - } - + /** Stops the requested YARN session cluster; an already absent cluster is treated as stopped. */ public ShutdownResponse shutdown(ShutdownRequest shutdownRequest) throws Exception { YarnClusterDescriptor clusterDescriptor = null; ClusterClient client = null; try { Configuration flinkConfig = loadDefaultConfiguration(shutdownRequest.flinkVersion().getFlinkHome()); - if (shutdownRequest.properties() != null) { - for (Map.Entry entry : shutdownRequest.properties().entrySet()) { - if (entry.getValue() != null) { - flinkConfig.setString(entry.getKey(), entry.getValue().toString()); - } + for (Map.Entry entry : shutdownRequest.properties().entrySet()) { + if (entry.getValue() != null) { + flinkConfig.setString(entry.getKey(), entry.getValue().toString()); } } - FlinkConfigurationOps.setIfPresent( - flinkConfig, YarnConfigOptions.APPLICATION_ID, shutdownRequest.clusterId()); - FlinkConfigurationOps.setIfPresent( - flinkConfig, DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); + flinkConfig.set(YarnConfigOptions.APPLICATION_ID, shutdownRequest.clusterId()); + flinkConfig.set(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); Tuple2 yarnClusterDescriptor = getYarnClusterDescriptor(flinkConfig); clusterDescriptor = yarnClusterDescriptor._2; - boolean shutdownState = + FinalApplicationStatus finalStatus = + clusterDescriptor + .getYarnClient() + .getApplicationReport(ApplicationId.fromString(shutdownRequest.clusterId())) + .getFinalApplicationStatus(); + boolean clusterRunning = FinalApplicationStatus.UNDEFINED.equals( - clusterDescriptor - .getYarnClient() - .getApplicationReport( - ApplicationId.fromString(shutdownRequest.clusterId())) - .getFinalApplicationStatus()); - if (shutdownState) { + finalStatus); + if (clusterRunning) { client = clusterDescriptor .retrieve(yarnClusterDescriptor._1) @@ -267,14 +151,13 @@ public ShutdownResponse shutdown(ShutdownRequest shutdownRequest) throws Excepti } logInfo( - "the " + "YARN application " + shutdownRequest.clusterId() - + "'s final status is " - + clusterDescriptor - .getYarnClient() - .getApplicationReport( - ConverterUtils.toApplicationId(shutdownRequest.clusterId())) - .getFinalApplicationStatus()); + + " had final status " + + finalStatus); + return new ShutdownResponse(shutdownRequest.clusterId()); + } catch (ApplicationNotFoundException e) { + // Shutdown is idempotent: a YARN application that no longer exists is already stopped. return new ShutdownResponse(shutdownRequest.clusterId()); } catch (Exception e) { logError("shutdown flink session fail in " + shutdownRequest.deployMode() + " mode"); @@ -283,4 +166,122 @@ public ShutdownResponse shutdown(ShutdownRequest shutdownRequest) throws Excepti Utils.close(client, clusterDescriptor); } } + + /** Adds the YARN session deployment target to a job submission configuration. */ + @Override + protected void setConfig(ResolvedSubmitRequest resolved, Configuration flinkConfig) { + super.setConfig(resolved, flinkConfig); + flinkConfig.set(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); + logEffectiveSubmitConfiguration(flinkConfig); + } + + /** Builds and submits a JobGraph to an existing YARN session cluster. */ + @Override + protected SubmitResponse doSubmit( + ResolvedSubmitRequest resolved, + Configuration flinkConfig) throws FlinkException { + SubmitRequest submitRequest = resolved.request(); + return execute( + () -> { + Tuple2 yarnClusterDescriptor = + getYarnClusterDescriptor(flinkConfig); + ApplicationId yarnClusterId = yarnClusterDescriptor._1; + YarnClusterDescriptor clusterDescriptor = yarnClusterDescriptor._2; + + FlinkJobGraphBuilder.Result job = null; + ClusterClient client = null; + try { + job = + buildJobGraph( + flinkConfig, + resolved, + resolved.getUserJarFile()); + + client = clusterDescriptor.retrieve(yarnClusterId).getClusterClient(); + String jobId = client.submitJob(job.jobGraph()).get().toString(); + logInfo("Flink job started: jobId=" + jobId + ", applicationId=" + yarnClusterId); + return new SubmitResponse( + yarnClusterId.toString(), + flinkConfig.toMap(), + jobId, + client.getWebInterfaceURL()); + } finally { + closeSubmissionResources( + job == null ? null : job.program(), + client, + clusterDescriptor); + } + }); + } + + /** Cancels a job while retaining its YARN session cluster. */ + @Override + protected CancelResponse doCancel( + CancelRequest cancelRequest, + Configuration flinkConfig) throws FlinkException { + flinkConfig.set(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); + return super.doCancel(cancelRequest, flinkConfig); + } + + /** Triggers a savepoint against an existing YARN session cluster. */ + @Override + protected SavepointResponse doTriggerSavepoint( + SavepointRequest request, + Configuration flinkConfig) throws FlinkException { + flinkConfig.set(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); + return super.doTriggerSavepoint(request, flinkConfig); + } + + /** Adds the remote distribution and local ship files required to deploy a session cluster. */ + private void deployClusterConfig(DeployRequest deployRequest, Configuration flinkConfig) { + ArrayList shipFiles = new ArrayList<>(); + shipFiles.add(deployRequest.flinkVersion().getFlinkHome() + "/lib"); + shipFiles.add(deployRequest.flinkVersion().getFlinkHome() + "/plugins"); + + RemoteWorkspace hdfsWorkspace = RemoteWorkspace.resolve(deployRequest.flinkVersion()); + flinkConfig.set(YarnConfigOptions.FLINK_DIST_JAR, hdfsWorkspace.flinkDistJar()); + flinkConfig.set(YarnConfigOptions.SHIP_FILES, shipFiles); + flinkConfig.set(DeploymentOptions.TARGET, YarnDeploymentTarget.SESSION.getName()); + flinkConfig.set( + DeploymentOptionsInternal.CONF_DIR, + deployRequest.flinkVersion().getFlinkHome() + "/conf"); + + logEffectiveSubmitConfiguration(flinkConfig); + } + + /** Returns a live existing session response, or {@code null} when redeployment is required. */ + private DeployResponse tryReuseExistingYarnSession( + DeployRequest deployRequest, + YarnClusterDescriptor clusterDescriptor) throws Exception { + try { + FinalApplicationStatus applicationStatus = + clusterDescriptor + .getYarnClient() + .getApplicationReport( + ApplicationId.fromString(deployRequest.clusterId())) + .getFinalApplicationStatus(); + if (FinalApplicationStatus.UNDEFINED != applicationStatus) { + return null; + } + try ( + ClusterClient yarnClient = + clusterDescriptor + .retrieve(ApplicationId.fromString(deployRequest.clusterId())) + .getClusterClient()) { + if (yarnClient.getWebInterfaceURL() == null) { + return null; + } + return new DeployResponse( + yarnClient.getWebInterfaceURL(), + yarnClient.getClusterId().toString(), + null); + } + } catch (ApplicationNotFoundException e) { + logInfo( + "YARN application " + + deployRequest.clusterId() + + " was not found; deploying a new session cluster"); + return null; + } + } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/bean/SessionClusterRestClientTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/bean/SessionClusterRestClientTest.java new file mode 100644 index 0000000000..61d2129bef --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/bean/SessionClusterRestClientTest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.bean; + +import org.apache.flink.client.deployment.application.ApplicationConfiguration; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.CoreOptions; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; + +class SessionClusterRestClientTest { + + @TempDir + Path tempDir; + + @Test + void submitsJarThroughRestApi() throws Exception { + File jobJar = tempDir.resolve("job.jar").toFile(); + Files.write(jobJar.toPath(), new byte[]{1, 2, 3}); + + AtomicReference uploadBody = new AtomicReference<>(); + AtomicReference runBody = new AtomicReference<>(); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/jars/upload", + exchange -> { + uploadBody.set(readBody(exchange)); + respond(exchange, "{\"filename\":\"/tmp/job.jar\",\"status\":\"success\"}"); + }); + server.createContext( + "/jars/job.jar/run", + exchange -> { + runBody.set(readBody(exchange)); + respond(exchange, "{\"jobid\":\"job-123\"}"); + }); + server.start(); + + try { + Configuration configuration = new Configuration(); + configuration.set(ApplicationConfiguration.APPLICATION_MAIN_CLASS, "example.Main"); + configuration.set( + ApplicationConfiguration.APPLICATION_ARGS, Arrays.asList("--name", "streampark")); + configuration.set(CoreOptions.DEFAULT_PARALLELISM, 2); + + String jobId = SessionClusterRestClient.submit( + "http://127.0.0.1:" + server.getAddress().getPort(), jobJar, configuration); + + assertThat(jobId).isEqualTo("job-123"); + assertThat(uploadBody.get()).contains("name=\"jarfile\"").contains("job.jar"); + assertThat(runBody.get()) + .contains("\"entryClass\":\"example.Main\"") + .contains("\"parallelism\":\"2\""); + } finally { + server.stop(0); + } + } + + private static String readBody(HttpExchange exchange) throws IOException { + return new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8); + } + + private static void respond(HttpExchange exchange, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, bytes.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(bytes); + } + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationBuilderTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationBuilderTest.java new file mode 100644 index 0000000000..6473632c97 --- /dev/null +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/configuration/FlinkConfigurationBuilderTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.streampark.flink.client.configuration; + +import org.apache.streampark.common.core.FlinkVersion; +import org.apache.streampark.common.enums.FlinkDeployMode; +import org.apache.streampark.common.enums.FlinkJobType; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; +import org.apache.streampark.flink.client.bean.SubmitRequestResolver; +import org.apache.streampark.flink.client.request.SubmitApplicationSpec; +import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse; + +import org.apache.flink.configuration.Configuration; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +class FlinkConfigurationBuilderTest { + + private static final String TEMPLATE = + "/logs/${jobName}/${jobname}/$jobName/$jobname/" + + "${jobId}/${jobid}/$jobId/$jobid"; + + @TempDir + Path tempDir; + + @Test + void expandJobVariables() { + Configuration configuration = new Configuration(); + configuration.setString("job.log.path", TEMPLATE); + + FlinkConfigurationBuilder.expandJobPlaceholders( + configuration, resolve(FlinkDeployMode.YARN_APPLICATION)); + + assertThat(configuration.getString("job.log.path", null)) + .isEqualTo("/logs/orders/orders/orders/orders/42/42/42/42"); + } + + @Test + void keepSharedVariables() { + Configuration configuration = new Configuration(); + configuration.setString("job.log.path", TEMPLATE); + + FlinkConfigurationBuilder.expandJobPlaceholders( + configuration, resolve(FlinkDeployMode.KUBERNETES_NATIVE_SESSION)); + + assertThat(configuration.getString("job.log.path", null)).isEqualTo(TEMPLATE); + } + + @Test + void dynamicPropertiesOverrideDefaults() throws Exception { + Path flinkHome = tempDir.resolve("flink"); + Files.createDirectories(flinkHome.resolve("lib")); + Files.createDirectories(flinkHome.resolve("conf")); + Files.createFile(flinkHome.resolve("lib/flink-dist_2.12-1.18.1.jar")); + Files.writeString( + flinkHome.resolve("conf/flink-conf.yaml"), "parallelism.default: 1\n"); + + Configuration configuration = + FlinkConfigurationBuilder.extract( + flinkHome.toString(), Map.of("parallelism.default", 4)); + + assertThat(configuration.getInteger("parallelism.default", -1)).isEqualTo(4); + } + + private static ResolvedSubmitRequest resolve(FlinkDeployMode deployMode) { + SubmitApplicationSpec job = + SubmitApplicationSpec.builder() + .jobType(FlinkJobType.FLINK_JAR) + .id(42L) + .appName("orders") + .build(); + SubmitRequest request = + new SubmitRequest( + new FlinkVersion("/tmp/flink-home"), + deployMode, + Collections.emptyMap(), + job, + null, + new ShadedBuildResponse("/tmp", "/tmp/job.jar"), + null); + return SubmitRequestResolver.resolve(request); + } +} diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestResolverTest.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestResolverTest.java index 649a9bd4d6..84bf35ee2e 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestResolverTest.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/SubmitRequestResolverTest.java @@ -24,9 +24,11 @@ import org.apache.streampark.common.enums.ApplicationType; import org.apache.streampark.common.enums.FlinkDeployMode; import org.apache.streampark.common.enums.FlinkJobType; +import org.apache.streampark.flink.client.bean.ResolvedSubmitRequest; import org.apache.streampark.flink.client.bean.SubmitRequestResolver; import org.apache.streampark.flink.client.request.SubmitApplicationSpec; import org.apache.streampark.flink.client.request.SubmitRequest; +import org.apache.streampark.flink.packer.pipeline.ShadedBuildResponse; import org.apache.flink.runtime.jobgraph.SavepointRestoreSettings; @@ -37,50 +39,52 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; class SubmitRequestResolverTest { private static final FlinkVersion FLINK_VERSION = new FlinkVersion("/tmp/flink-home"); @Test - void appMainForFlinkSqlJob() { + void resolveFlinkSqlMainClass() { SubmitRequest request = createRequest(FlinkJobType.FLINK_SQL, null); - assertThat(SubmitRequestResolver.applicationMain(request)) + assertThat(SubmitRequestResolver.resolve(request).getJobMainClass()) .isEqualTo(Constants.STREAMPARK_FLINKSQL_CLIENT_CLASS); } @Test - void appMainForPyFlinkJob() { + void resolvePyFlinkMainClass() { SubmitRequest request = createRequest(FlinkJobType.PYFLINK, null); - assertThat(SubmitRequestResolver.applicationMain(request)) + assertThat(SubmitRequestResolver.resolve(request).getJobMainClass()) .isEqualTo(Constants.PYTHON_FLINK_DRIVER_CLASS_NAME); } @Test - void appMainForFlinkJarFromJsonConf() { + void resolveJarMainClassFromJson() { String mainClass = "org.apache.flink.streaming.examples.windowing.TopSpeedWindowing"; String appConf = String.format( "json://{\"%s\":\"%s\"}", FlinkOptions.APPLICATION_MAIN_CLASS.key(), mainClass); SubmitRequest request = createRequest(FlinkJobType.FLINK_JAR, appConf); - assertThat(SubmitRequestResolver.applicationMain(request)).isEqualTo(mainClass); + assertThat(SubmitRequestResolver.resolve(request).getJobMainClass()).isEqualTo(mainClass); } @Test - void savepointRestoreSettingsWithoutSavepoint() { + void returnNoRestoreSettings() { SubmitRequest request = createRequest(FlinkJobType.FLINK_JAR, null); - assertThat(SubmitRequestResolver.savepointRestoreSettings(request)) + assertThat(SubmitRequestResolver.resolve(request).savepointRestoreSettings()) .isEqualTo(SavepointRestoreSettings.none()); } @Test - void appPropertiesFromJsonConf() { + void extractPropertiesFromJson() { String propertyKey = FlinkOptions.PROPERTY_PREFIX.value() + "parallelism.default"; String appConf = "json://{\"" + propertyKey + "\":\"8\"}"; SubmitApplicationSpec application = SubmitApplicationSpec.builder() .jobType(FlinkJobType.FLINK_JAR) + .appName("test-job") .appConf(appConf) .build(); SubmitRequest request = @@ -90,13 +94,36 @@ void appPropertiesFromJsonConf() { Collections.emptyMap(), application, null, - null, + buildResult(), null); - assertThat(SubmitRequestResolver.applicationProperties(request)) + assertThat(SubmitRequestResolver.resolve(request).jobProperties()) .containsEntry("parallelism.default", "8"); } + @Test + void keepResolvedConfigImmutable() { + String propertyKey = FlinkOptions.PROPERTY_PREFIX.value() + "parallelism.default"; + String optionKey = FlinkOptions.OPTION_PREFIX.value() + "detached"; + SubmitRequest request = + createRequest( + FlinkJobType.FLINK_JAR, + "json://{\"" + + propertyKey + + "\":\"8\",\"" + + optionKey + + "\":\"true\"}"); + + ResolvedSubmitRequest resolved = SubmitRequestResolver.resolve(request); + + assertThat(resolved.jobProperties()) + .containsExactly(Map.entry("parallelism.default", "8")); + assertThat(resolved.jobOptions()) + .containsExactly(Map.entry("detached", "true")); + assertThatThrownBy(() -> resolved.jobProperties().put("new.key", "value")) + .isInstanceOf(UnsupportedOperationException.class); + } + @Test void copySerializableIntegers() { Map properties = new HashMap<>(); @@ -117,13 +144,13 @@ void copySerializableIntegers() { } @Test - void returnNullWithoutFlinkSql() { + void returnNullWithoutSql() { SubmitRequest request = createRequest(FlinkJobType.FLINK_SQL, null); - assertThat(SubmitRequestResolver.flinkSql(request)).isNull(); + assertThat(SubmitRequestResolver.resolve(request).getFlinkSqlContent()).isNull(); } @Test - void flinkSqlShouldReadFromExtraParameter() { + void readSqlFromExtraParameters() { Map extra = new HashMap<>(); extra.put(ApplicationOptions.SQL.key(), "select 1"); SubmitRequest request = @@ -131,23 +158,26 @@ void flinkSqlShouldReadFromExtraParameter() { FLINK_VERSION, FlinkDeployMode.YARN_APPLICATION, Collections.emptyMap(), - SubmitApplicationSpec.builder().jobType(FlinkJobType.FLINK_SQL).build(), - null, + SubmitApplicationSpec.builder() + .jobType(FlinkJobType.FLINK_SQL) + .appName("test-job") + .build(), null, + buildResult(), extra); - assertThat(SubmitRequestResolver.flinkSql(request)).isEqualTo("select 1"); + assertThat(SubmitRequestResolver.resolve(request).getFlinkSqlContent()).isEqualTo("select 1"); } @Test - void extraParametersShouldDefaultToEmptyMap() { + void defaultExtraParametersToEmptyMap() { SubmitRequest request = createRequest(FlinkJobType.FLINK_JAR, null); assertThat(request.extraParameter()).isEmpty(); } @Test - void defaultAllowNonRestoredState() { + void disallowNonRestoredStateByDefault() { SubmitRequest request = createRequest(FlinkJobType.FLINK_JAR, null); - assertThat(SubmitRequestResolver.allowNonRestoredState(request)).isFalse(); + assertThat(SubmitRequestResolver.resolve(request).isNonRestoredStateAllowed()).isFalse(); } @Test @@ -170,10 +200,52 @@ void preserveYamlFormat() { assertThat(request.standardYaml()).isTrue(); } + @Test + void rejectMissingJobName() { + SubmitApplicationSpec application = + SubmitApplicationSpec.builder().jobType(FlinkJobType.FLINK_JAR).build(); + SubmitRequest request = + new SubmitRequest( + FLINK_VERSION, + FlinkDeployMode.YARN_APPLICATION, + Collections.emptyMap(), + application, + null, + buildResult(), + null); + + assertThatThrownBy(() -> SubmitRequestResolver.resolve(request)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Flink job name must not be blank"); + } + + @Test + void rejectMissingUserJar() { + SubmitApplicationSpec job = + SubmitApplicationSpec.builder() + .jobType(FlinkJobType.FLINK_JAR) + .appName("test-job") + .build(); + SubmitRequest request = + new SubmitRequest( + FLINK_VERSION, + FlinkDeployMode.YARN_APPLICATION, + Collections.emptyMap(), + job, + null, + new ShadedBuildResponse("/tmp", null), + null); + + assertThatThrownBy(() -> SubmitRequestResolver.resolve(request)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Built user JAR path must not be blank"); + } + private static SubmitRequest createRequest(FlinkJobType jobType, String appConf) { SubmitApplicationSpec application = SubmitApplicationSpec.builder() .jobType(jobType) + .appName("test-job") .applicationType(ApplicationType.APACHE_FLINK) .appConf(appConf) .build(); @@ -183,7 +255,11 @@ private static SubmitRequest createRequest(FlinkJobType jobType, String appConf) Collections.emptyMap(), application, null, - null, + buildResult(), null); } + + private static ShadedBuildResponse buildResult() { + return new ShadedBuildResponse("/tmp", "/tmp/job.jar"); + } } diff --git a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/YarnPerJobTestCase.java b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/YarnPerJobTestCase.java index 9a5fd21084..7f0550a46e 100644 --- a/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/YarnPerJobTestCase.java +++ b/streampark-flink/streampark-flink-client/streampark-flink-client-core/src/test/java/org/apache/streampark/flink/client/test/YarnPerJobTestCase.java @@ -58,7 +58,7 @@ import static org.assertj.core.api.Assertions.assertThat; -/** perJob to submit jobs programmatically */ +/** Compatibility harness for programmatic submission through Flink's legacy YARN per-job API. */ public final class YarnPerJobTestCase { private static final Logger LOG = @@ -75,32 +75,6 @@ public final class YarnPerJobTestCase { private YarnPerJobTestCase() { } - private static void ensureInitialized() { - if (deployInternalMethod != null) { - return; - } - String flinkHome = Objects.requireNonNull(System.getenv("FLINK_HOME"), "FLINK_HOME must be set"); - LOG.info("flinkHome: {}", flinkHome); - flinkDefaultConfiguration = GlobalConfiguration.loadConfiguration(flinkHome + "/conf"); - try { - customCommandLines = - CliFrontend.loadCustomCommandLines(flinkDefaultConfiguration, flinkHome + "/conf"); - Class[] paramClass = - new Class[]{ - ClusterSpecification.class, - String.class, - String.class, - JobGraph.class, - boolean.class - }; - deployInternalMethod = - YarnClusterDescriptor.class.getDeclaredMethod("deployInternal", paramClass); - deployInternalMethod.setAccessible(true); - } catch (Exception e) { - throw new IllegalStateException("Failed to initialize YARN integration harness", e); - } - } - @Test void verifyYarnPerJobHarnessCompatibility() throws Exception { Method deployInternal = @@ -131,23 +105,7 @@ void verifyYarnPerJobHarnessCompatibility() throws Exception { } } - @SuppressWarnings("unchecked") - private static ClusterClientProvider deployInternal( - YarnClusterDescriptor clusterDescriptor, - ClusterSpecification clusterSpecification, - String applicationName, - String yarnClusterEntrypoint, - JobGraph jobGraph, - Boolean detached) throws Exception { - return (ClusterClientProvider) deployInternalMethod.invoke( - clusterDescriptor, - clusterSpecification, - applicationName, - yarnClusterEntrypoint, - jobGraph, - detached); - } - + /** Runs a manual submission against the Flink installation identified by {@code FLINK_HOME}. */ public static void main(String[] args) throws Exception { ensureInitialized(); String flinkHome = System.getenv("FLINK_HOME"); @@ -246,4 +204,49 @@ public static void main(String[] args) throws Exception { clusterDescriptor.close(); } } + + /** Initializes the reflective Flink YARN entry point only when the manual harness is used. */ + private static void ensureInitialized() { + if (deployInternalMethod != null) { + return; + } + String flinkHome = Objects.requireNonNull(System.getenv("FLINK_HOME"), "FLINK_HOME must be set"); + LOG.info("flinkHome: {}", flinkHome); + flinkDefaultConfiguration = GlobalConfiguration.loadConfiguration(flinkHome + "/conf"); + try { + customCommandLines = + CliFrontend.loadCustomCommandLines(flinkDefaultConfiguration, flinkHome + "/conf"); + Class[] paramClass = + new Class[]{ + ClusterSpecification.class, + String.class, + String.class, + JobGraph.class, + boolean.class + }; + deployInternalMethod = + YarnClusterDescriptor.class.getDeclaredMethod("deployInternal", paramClass); + deployInternalMethod.setAccessible(true); + } catch (Exception e) { + throw new IllegalStateException("Failed to initialize YARN integration harness", e); + } + } + + /** Invokes the non-public YARN per-job deployment method retained by supported Flink 1.x. */ + @SuppressWarnings("unchecked") + private static ClusterClientProvider deployInternal( + YarnClusterDescriptor clusterDescriptor, + ClusterSpecification clusterSpecification, + String jobName, + String yarnClusterEntrypoint, + JobGraph jobGraph, + Boolean detached) throws Exception { + return (ClusterClientProvider) deployInternalMethod.invoke( + clusterDescriptor, + clusterSpecification, + jobName, + yarnClusterEntrypoint, + jobGraph, + detached); + } } diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/package-info.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/package-info.java index a25cd5fa04..10f74afe6e 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/package-info.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/configuration/package-info.java @@ -20,7 +20,7 @@ * *

StreamPark application options and native Flink configuration remain separate until the * initializer creates a {@link org.apache.streampark.flink.configuration.FlinkRuntimeConfiguration} - * for a concrete job. {@link FlinkConfigurationUtils} + * for a concrete job. {@link FlinkConfigurationLoader} * normalizes Flink's legacy and standard YAML file formats before those views are assembled. * *

Installations before Flink 1.19 use flat {@code flink-conf.yaml}. Flink 1.19 and 1.20 prefer @@ -30,4 +30,4 @@ */ package org.apache.streampark.flink.configuration; -import org.apache.streampark.common.util.FlinkConfigurationUtils; +import org.apache.streampark.common.util.FlinkConfigurationLoader; diff --git a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializer.java b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializer.java index b731345e06..dc8ce882c7 100644 --- a/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializer.java +++ b/streampark-flink/streampark-flink-shims/streampark-flink-shims-base/src/main/java/org/apache/streampark/flink/core/FlinkTableInitializer.java @@ -25,7 +25,7 @@ import org.apache.streampark.common.configuration.option.ApplicationOptions; import org.apache.streampark.common.enums.PlannerType; import org.apache.streampark.common.util.DeflaterUtils; -import org.apache.streampark.common.util.FlinkConfigurationUtils; +import org.apache.streampark.common.util.FlinkConfigurationLoader; import org.apache.streampark.common.util.StreamParkLoggerFactory; import org.apache.streampark.flink.configuration.FlinkJobParameters; import org.apache.streampark.flink.configuration.FlinkRuntimeConfiguration; @@ -145,7 +145,7 @@ private EnvironmentSettings.Builder buildEnvSettings( // application YAML continues to use the engine-neutral parser. builder.withConfiguration( org.apache.flink.configuration.Configuration.fromMap( - FlinkConfigurationUtils.loadConfigurationFromString( + FlinkConfigurationLoader.loadConfigurationFromString( DeflaterUtils.unzipString(flinkConf), parameters.get(FlinkOptions.FLINK_CONFIGURATION_STANDARD_YAML)))); From 41abdfc652508468c3b27a3e3fd9ef460d4d3d33 Mon Sep 17 00:00:00 2001 From: benjobs Date: Fri, 4 Sep 2026 13:00:09 +0800 Subject: [PATCH 9/9] [CI] Keep retired E2E check mergeable --- .github/workflows/e2e.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000000..6b016a8e74 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: E2E Required Check Compatibility + +on: + pull_request: + push: + branches: + - dev + +# The retired E2E workflow remains a required check on dev until the .asf.yaml change in this PR +# is merged and synchronized to GitHub branch protection. Remove this compatibility workflow after +# that synchronization completes. +jobs: + result: + name: E2E - Result + runs-on: ubuntu-latest + steps: + - name: Report retired E2E status + run: echo "The legacy E2E suite is retired for the StreamPark 3.0 architecture transition."