diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000000..dd84ea7824f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,38 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + - OS: [e.g. iOS] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + +**Smartphone (please complete the following information):** + - Device: [e.g. iPhone6] + - OS: [e.g. iOS8.1] + - Browser [e.g. stock browser, safari] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000000..bbcbbe7d615 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index d6456956733..00000000000 --- a/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/README.md b/README.md index 421e36397fa..fa7787dd2a8 100644 --- a/README.md +++ b/README.md @@ -1,48 +1,78 @@ -## Introduction +Linkis +============ -WilLink is a collection of middleware services for financial-level big data platforms. Connected to WilLink foreground system, a user can obtain the capabilities of financial-level multi-tenant, resource management and permission isolation. Besides, unified variables, functions and source file management across multiple foreground systems are realized. In addition, it empowers users to get lifecycle management capabilities for high-concurrency, high-performance and high-availability jobs. +[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) -Based on the microservices architecture, WilLink also supports plug-in connection to various back-end big data computing and storage engines, allowing users’ data development/display tool system instantly upgrading to financial level. +English | [中文](docs/zh_CN/README.md) -WilLink, the link between user and data platform. +## Introduction + +Linkis helps easily connect to various back-end computation/storage engines(Spark, Python, HBase ...), exposes various interfaces(REST, WebSocket, Java ...), with multi-tenancy, high performance, and resource control. + +Linkis connects with computation/storage engines(Spark, Hive, Python and HBase), exposes REST/WS interface, and executes multi-language jobs(SQL, Pyspark, HiveQL and Scala), as a data middleware. + +Based on the microservices architecture, Linkis provides enterprise-level features of financial-level multi-tenant isolation, resource management and access control. It also offers convenient support to manage unified variables, UDFs, functions and resource files. it is also guaranteed with sophisticated task/job lifecycle management capabilities under high-concurrency, high-performance and high-availability scenarios. + +![linkis-intro-01](https://github.com/WeBankFinTech/Linkis/blob/master/images/linkis-intro-01.png?raw=true) +![linkis-intro-03](https://github.com/WeBankFinTech/Linkis/blob/master/images/linkis-intro-03.png?raw=true) ## Features -- Unified Job Execution Services: A distributed REST/WebSocket container designed for receiving data analysis and exploration requests submitted by foreground system. +- Unified Job Execution Services: A distributed REST/WebSocket service for processing scripts execution requests from user. + + Available computation engines so far: Spark, Python, TiSpark, Hive and Shell. + + Available languages so far: SparkSQL, Spark Scala, PySpark, R, Python, HQL and Shell. + +- Resource Management Services: Available for real-time control/limit of resource usage from both perspectives of amount and load for both systems and users. With dynamic charts of resource statistics, it is convenient to monitor and manage resource usage for systems and users. + + Available resource types so far: Yarn queue resources, server(CPU and memory), number of concurrent instances per user. + + +- Application Management Services: Manages global user applications, including offline batch applications, interactive query applications and real-time streaming applications. Also provides powerful reusability especially for offline and interactive applications, with complete lifecycle management which automatically releases idle applications for users. + +- Unified Storage Services: The generic IO architecture can quickly integrate with various storage systems and provide a unified invokable entrance. It is also highly integrated with most common data formats and easy to use. + +- Unified Context Services: Unite resources files of users and systems (JAR, ZIP, Properties). With unified management of arguments/variables for users, systems and engines, it is achieved that modification in random place will reflect in all the other places automatically. + + - Currently supported compute engines: Spark, Python, TiSpark, Hive and Shell, etc. +- Material Library: System and user-level material management, capable of sharing, transferring materials and automatic lifecycle management. - Currently supported script languages: SparkSQL, Spark Scala, PySpark, R, Python, HQL and Shell, etc. +- Metadata Services: Real-time display of dataset table structure and partitions. -- Resource Management Services: Provide real-time control of each user's resources usage and dynamic resources charts and thus make it easy for users to view and manage their own resources. + **Compared with similar systems** - Currently supported resources type: Yarn queue resources, server(CPU and memory), number of concurrent users, etc. +![introduction01](docs/en_US/images/introduction/introduction01.png) -- Application Management Services: Manage all user applications across all systems, including offline batch applications, interactive query applications as well as live streaming applications, providing powerful reusability for offline and interactive application and providing application lifecycle management that automatically frees up user idle applications. +# Documentations： -- Unified Storage Services: The generic IO architecture can quickly connected with various storage systems and provide a uniform call entry. Besides, it is highly integrated, easy to use and supports all common data formats. +[Linkis, make big data easier](docs/en_US/ch3/Linkis_Introduction.md) -- Unified Context Services: Unify user and system resources files (JAR, ZIP, Properties). User, system, parameters and variables of compute engine are managed in a unified manner. Therefore, once there is a change, the rest that associated with it will change accordingly. +[Linkis Quick Deploy](docs/en_US/ch1/deploy.md) -- Material Library: System and user-level material management, it is capable of sharing, transferring materials and supports automatic lifecycle management. +[Linkis Quick Start & Java SDK documentation](docs/en_US/ch3/Linkis_Java_SDK_doc.md) -- Metadata Services: Real-time display of dataset table structure and partition. +[HTTP APIs for frontend applications](docs/en_US/ch3/Linkis_HTTP_API_Doc.md) -## QuickStart +[WebSocket APIs for frontend applications](docs/en_US/ch3/Linkis_WebSocket_API_Doc.md) -Read the Quick Start. +[How to adapt Linkis with a new computation or storage engine](docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md) -## Architecture +---- -## RoadMap +# Architecture： -## Contributing +![introduction02](./docs/en_US/images/introduction/introduction02.png) -## Adopters +---- ## Communication +If you desire immediate response, please kindly raise issues to us or scan the below QR code by WeChat and QQ to join our group: +
+![introduction05](docs/en_US/images/introduction/introduction05.png) ## License -Scriptest is under the Apache 2.0 license. See the [LICENSE ](http://www.apache.org/licenses/LICENSE-2.0)file for details. +Linkis is under the Apache 2.0 license. See the [LICENSE ](http://www.apache.org/licenses/LICENSE-2.0)file for details. diff --git a/assembly/pom.xml b/assembly/pom.xml index 0714ae7f738..2c887aba545 100644 --- a/assembly/pom.xml +++ b/assembly/pom.xml @@ -21,11 +21,11 @@ com.webank.wedatasphere.linkis linkis - 0.5.0-SNAPSHOT + 0.8.0 4.0.0 - assembly + wedatasphere-linkis @@ -50,7 +50,7 @@ org.apache.maven.plugins maven-assembly-plugin - 3.1.0 + 2.3 dist @@ -59,9 +59,13 @@ single - - src/main/assembly/assembly.xml - + false + wedatasphere-linkis-${linkis.version}-dist + false + false + + src/main/assembly/assembly.xml + diff --git a/bin/install.sh b/bin/install.sh index 4a125d1293a..5b311875f0d 100644 --- a/bin/install.sh +++ b/bin/install.sh @@ -1,7 +1,7 @@ #!/bin/sh -workDir=`dirname $0` -workDir=`cd ${workDir};pwd` +shellDir=`dirname $0` +workDir=`cd ${shellDir}/..;pwd` @@ -29,45 +29,128 @@ else exit 1 fi +function isSuccess(){ +if [ $? -ne 0 ]; then + echo "Failed to " + $1 + exit 1 +else + echo "Succeed to" + $1 +fi +} + +function checkPythonAndJava(){ + python --version + isSuccess "execute python --version" + java -version + isSuccess "execute java --version" +} + +function checkHadoopAndHive(){ + hdfs version + isSuccess "execute hdfs version" + hive --help + isSuccess "execute hive -h" +} + +function checkSpark(){ + spark-submit --version + isSuccess "execute spark-submit --version" +} + ##install env:expect, sudo yum install -y expect +isSuccess "install expect" ##load config -echo "step1:load config" +echo "step1:load config " source ${workDir}/conf/config.sh source ${workDir}/conf/db.sh +isSuccess "load config" local_host="`hostname --fqdn`" +##env check +echo "Please enter the mode selection such as: 1" +echo " 1: Lite(精简版)" +echo " 2: Simple(简单版)" +echo " 3: Standard(标准版)" +echo "" + +INSTALL_MODE=1 + +read -p "Please input the choice:" idx +if [[ '1' = "$idx" ]];then + INSTALL_MODE=1 + echo "You chose Lite installation mode" + checkPythonAndJava +elif [[ '2' = "$idx" ]];then + INSTALL_MODE=2 + echo "You chose Simple installation mode" + checkPythonAndJava + checkHadoopAndHive +elif [[ '3' = "$idx" ]];then + INSTALL_MODE=3 + echo "You chose Standard installation mode" + checkPythonAndJava + checkHadoopAndHive + checkSpark +else + echo "no choice,exit!" + exit 1 +fi + + +##env check +echo "Do you empty Linkis table information in the database?" +echo " 1: Rebuild the table(重新建表)" +echo " 2: Do not execute table-building statements(不执行建表语句)" +echo "" + +MYSQL_INSTALL_MODE=1 + +read -p "Please input the choice:" idx +if [[ '1' = "$idx" ]];then + MYSQL_INSTALL_MODE=1 + echo "You chose Rebuild the table" +elif [[ '2' = "$idx" ]];then + MYSQL_INSTALL_MODE=2 + echo "You chose not execute table-building statements" +else + echo "no choice,exit!" + exit 1 +fi + ##stop server -echo "step2,stop server" -sh ${workDir}/bin/stop_all.sh +#echo "step2,stop server" +#sh ${workDir}/bin/stop-all.sh ##Eurkea install SERVERNAME=eureka SERVER_IP=$EUREKA_INSTALL_IP SERVER_PORT=$EUREKA_PORT SERVER_HOME=$LINKIS_INSTALL_HOME -EUEKEA_URL=http://$EUREKA_INSTALL_IP:$EUREKA_PORT/eureka/ echo "$SERVERNAME-step1: create dir" if test -z "$SERVER_IP" then SERVER_IP=$local_host fi - +EUREKA_URL=http://$SERVER_IP:$EUREKA_PORT/eureka/ if ! ssh $SERVER_IP test -e $SERVER_HOME; then ssh $SERVER_IP "sudo mkdir -p $SERVER_HOME;sudo chown -R $deployUser:$deployUser $SERVER_HOME" + isSuccess "create the dir of $SERVER_HOME" fi echo "$SERVERNAME-step2:copy install package" scp ${workDir}/share/springcloud/$SERVERNAME/$SERVERNAME.zip $SERVER_IP:$SERVER_HOME +isSuccess "copy $SERVERNAME" ssh $SERVER_IP "cd $SERVER_HOME/;rm -rf eureka;unzip $SERVERNAME.zip > /dev/null" echo "$SERVERNAME-step3:subsitution conf" eureka_conf_path=$SERVER_HOME/$SERVERNAME/conf/application-$SERVERNAME.yml ssh $SERVER_IP "sed -i \"s#port:.*#port: $SERVER_PORT#g\" $eureka_conf_path" -ssh $SERVER_IP "sed -i \"s#defaultZone:.*#defaultZone: $EUEKEA_URL#g\" $eureka_conf_path" +ssh $SERVER_IP "sed -i \"s#defaultZone:.*#defaultZone: $EUREKA_URL#g\" $eureka_conf_path" ssh $SERVER_IP "sed -i \"s#hostname:.*#hostname: $SERVER_IP#g\" $eureka_conf_path" +isSuccess "subsitution conf of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" ##Eurkea install end @@ -83,23 +166,29 @@ fi if ! ssh $SERVER_IP test -e $SERVER_HOME; then ssh $SERVER_IP "sudo mkdir -p $SERVER_HOME;sudo chown -R $deployUser:$deployUser $SERVER_HOME" + isSuccess "create the dir of $SERVERNAME" fi echo "$SERVERNAME-step2:copy install package" scp ${workDir}/share/$PACKAGE_DIR/$SERVERNAME.zip $SERVER_IP:$SERVER_HOME +isSuccess "copy ${SERVERNAME}.zip" ssh $SERVER_IP "cd $SERVER_HOME/;rm -rf $SERVERNAME-bak; mv -f $SERVERNAME $SERVERNAME-bak" ssh $SERVER_IP "cd $SERVER_HOME/;unzip $SERVERNAME.zip > /dev/null" +isSuccess "unzip ${SERVERNAME}.zip" if [ "$SERVERNAME" != "linkis-gateway" ] then scp ${workDir}/share/linkis/module/module.zip $SERVER_IP:$SERVER_HOME + isSuccess "cp module.zip" ssh $SERVER_IP "cd $SERVER_HOME/;rm -rf modulebak;mv -f module modulebak;" ssh $SERVER_IP "cd $SERVER_HOME/;unzip module.zip > /dev/null;cp module/lib/* $SERVER_HOME/$SERVERNAME/lib/" + isSuccess "unzip module.zip" fi echo "$SERVERNAME-step3:subsitution conf" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/application.yml ssh $SERVER_IP "sed -i \"s#port:.*#port: $SERVER_PORT#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#defaultZone:.*#defaultZone: $EUEKEA_URL#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#defaultZone:.*#defaultZone: $EUREKA_URL#g\" $SERVER_CONF_PATH" ssh $SERVER_IP "sed -i \"s#hostname:.*#hostname: $SERVER_IP#g\" $SERVER_CONF_PATH" +isSuccess "subsitution conf of $SERVERNAME" } ##function end @@ -117,6 +206,7 @@ SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties ssh $SERVER_IP "sed -i \"s#wds.linkis.ldap.proxy.url.*#wds.linkis.ldap.proxy.url=$LDAP_URL#g\" $SERVER_CONF_PATH" ssh $SERVER_IP "sed -i \"s#wds.linkis.ldap.proxy.baseDN.*#wds.linkis.ldap.proxy.baseDN=$LDAP_BASEDN#g\" $SERVER_CONF_PATH" ssh $SERVER_IP "sed -i \"s#wds.linkis.gateway.admin.user.*#wds.linkis.gateway.admin.user=$deployUser#g\" $SERVER_CONF_PATH" +isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" ##GateWay Install end @@ -134,31 +224,14 @@ SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH" ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g\" $SERVER_CONF_PATH" ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#wds.linkis.workspace.filesystem.localuserrootpath.*#wds.linkis.workspace.filesystem.localuserrootpath=$WORKSPACE_PATH#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#wds.linkis.workspace.filesystem.hdfsuserrootpath.prefix.*#wds.linkis.workspace.filesystem.hdfsuserrootpath.prefix=$HDFS_PATH#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.workspace.filesystem.localuserrootpath.*#wds.linkis.workspace.filesystem.localuserrootpath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.workspace.filesystem.hdfsuserrootpath.prefix.*#wds.linkis.workspace.filesystem.hdfsuserrootpath.prefix=$HDFS_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" +isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" ##publicservice end -##linkis-database install -PACKAGE_DIR=linkis/linkis-database -SERVERNAME=linkis-database -SERVER_IP=$DATABASE_INSTALL_IP -SERVER_PORT=$DATABASE_PORT -SERVER_HOME=$LINKIS_INSTALL_HOME -###install dir -installPackage -###update linkis.properties -echo "$SERVERNAME-step4:update linkis conf" -SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#hive.meta.url.*#hive.meta.url=$HIVE_META_URL#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#hive.meta.user.*#hive.meta.user=$HIVE_META_USER#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#hive.meta.password.*#hive.meta.user=$HIVE_META_PASSWORD#g\" $SERVER_CONF_PATH" -echo "<----------------$SERVERNAME:end------------------->" -##database end + ##ResourceManager install @@ -175,14 +248,27 @@ SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH" ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g\" $SERVER_CONF_PATH" ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "rm $SERVER_HOME/$SERVERNAME/lib/json4s-*3.5.3.jar" +echo "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" ##ResourceManager install end -##SparkEM install -PACKAGE_DIR=linkis/ujes/spark -SERVERNAME=linkis-ujes-spark-enginemanager -SERVER_IP=$SPARK_INSTALL_IP -SERVER_PORT=$SPARK_EM_PORT +##init db +if [[ '1' = "$MYSQL_INSTALL_MODE" ]];then + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB -e "source ${workDir}/db/linkis_ddl.sql" + isSuccess "source linkis_ddl.sql" + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB -e "source ${workDir}/db/linkis_dml.sql" + isSuccess "source linkis_dml.sql" + echo "Rebuild the table" +fi + + + +##PythonEM install +PACKAGE_DIR=linkis/ujes/python +SERVERNAME=linkis-ujes-python-enginemanager +SERVER_IP=$PYTHON_INSTALL_IP +SERVER_PORT=$PYTHON_EM_PORT SERVER_HOME=$LINKIS_INSTALL_HOME ###install dir installPackage @@ -190,22 +276,47 @@ installPackage echo "$SERVERNAME-step4:update linkis conf" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties ssh $SERVER_IP "sed -i \"s#wds.linkis.enginemanager.sudo.script.*#wds.linkis.enginemanager.sudo.script=$SERVER_HOME/$SERVERNAME/bin/rootScript.sh#g\" $SERVER_CONF_PATH" +isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" -##SparkEM install end -##SparkEntrance install -PACKAGE_DIR=linkis/ujes/spark -SERVERNAME=linkis-ujes-spark-entrance -SERVER_PORT=$SPARK_ENTRANCE_PORT + +##PythonEntrance install +PACKAGE_DIR=linkis/ujes/python +SERVERNAME=linkis-ujes-python-entrance +SERVER_PORT=$PYTHON_ENTRANCE_PORT ###install dir installPackage ###update linkis.properties echo "$SERVERNAME-step4:update linkis conf" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -ssh $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$USER_LOG_PATH#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_STORE_PATH#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH" +isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" -##SparkEntrance install end +##PythonEntrance install end + +if [[ '1' = "$INSTALL_MODE" ]];then + echo "Lite install end" + exit 0 +fi + +##linkis-database install +PACKAGE_DIR=linkis/linkis-database +SERVERNAME=linkis-database +SERVER_IP=$DATABASE_INSTALL_IP +SERVER_PORT=$DATABASE_PORT +SERVER_HOME=$LINKIS_INSTALL_HOME +###install dir +installPackage +###update linkis.properties +echo "$SERVERNAME-step4:update linkis conf" +SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties +ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.url.*#wds.linkis.server.mybatis.datasource.url=jdbc:mysql://${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB}?characterEncoding=UTF-8#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.username.*#wds.linkis.server.mybatis.datasource.username=$MYSQL_USER#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.server.mybatis.datasource.password.*#wds.linkis.server.mybatis.datasource.password=$MYSQL_PASSWORD#g\" $SERVER_CONF_PATH" +isSuccess "subsitution linkis.properties of $SERVERNAME" +echo "<----------------$SERVERNAME:end------------------->" +##database end ##HiveEM install PACKAGE_DIR=linkis/ujes/hive @@ -219,6 +330,8 @@ installPackage echo "$SERVERNAME-step4:update linkis conf" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties ssh $SERVER_IP "sed -i \"s#wds.linkis.enginemanager.sudo.script.*#wds.linkis.enginemanager.sudo.script=$SERVER_HOME/$SERVERNAME/bin/rootScript.sh#g\" $SERVER_CONF_PATH" +isSuccess "subsitution linkis.properties of $SERVERNAME" +ssh $SERVER_IP "rm $SERVER_HOME/$SERVERNAME/lib/servlet-api-2.5.jar" echo "<----------------$SERVERNAME:end------------------->" ##HiveEM install end @@ -231,77 +344,52 @@ installPackage ###update linkis.properties echo "$SERVERNAME-step4:update linkis conf" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -ssh $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$USER_LOG_PATH#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_STORE_PATH#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_SET_ROOT_PATH#g\" $SERVER_CONF_PATH" +isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" ##HiveEntrance install end +if [[ '2' = "$INSTALL_MODE" ]];then + echo "Simple install end" + exit 0 +fi -##PythonEM install -PACKAGE_DIR=linkis/ujes/python -SERVERNAME=linkis-ujes-python-enginemanager -SERVER_IP=$PYTHON_INSTALL_IP -SERVER_PORT=$PYTHON_EM_PORT -SERVER_HOME=$LINKIS_INSTALL_HOME -###install dir -installPackage -###update linkis.properties -echo "$SERVERNAME-step4:update linkis conf" -SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -ssh $SERVER_IP "sed -i \"s#wds.linkis.enginemanager.sudo.script.*#wds.linkis.enginemanager.sudo.script=$SERVER_HOME/$SERVERNAME/bin/rootScript.sh#g\" $SERVER_CONF_PATH" -echo "<----------------$SERVERNAME:end------------------->" - - -##PythonEntrance install -PACKAGE_DIR=linkis/ujes/python -SERVERNAME=linkis-ujes-python-entrance -SERVER_PORT=$PYTHON_ENTRANCE_PORT -###install dir -installPackage -###update linkis.properties -echo "$SERVERNAME-step4:update linkis conf" -SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -ssh $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$USER_LOG_PATH#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_STORE_PATH#g\" $SERVER_CONF_PATH" -echo "<----------------$SERVERNAME:end------------------->" -##PythonEntrance install end +if [[ '3' != "$INSTALL_MODE" ]];then + exit 0 +fi -##pipelineEM install -PACKAGE_DIR=linkis/ujes/pipeline -SERVERNAME=linkis-ujes-pipeline-enginemanager -SERVER_IP=$PEPELINE_INSTALL_IP -SERVER_PORT=$PEPELINE_EM_PORT +##SparkEM install +PACKAGE_DIR=linkis/ujes/spark +SERVERNAME=linkis-ujes-spark-enginemanager +SERVER_IP=$SPARK_INSTALL_IP +SERVER_PORT=$SPARK_EM_PORT SERVER_HOME=$LINKIS_INSTALL_HOME ###install dir installPackage ###update linkis.properties echo "$SERVERNAME-step4:update linkis conf" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties +ENGINE_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis-engine.properties ssh $SERVER_IP "sed -i \"s#wds.linkis.enginemanager.sudo.script.*#wds.linkis.enginemanager.sudo.script=$SERVER_HOME/$SERVERNAME/bin/rootScript.sh#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.enginemanager.core.jar.*#wds.linkis.enginemanager.core.jar=$SERVER_HOME/$SERVERNAME/lib/linkis-ujes-spark-engine-$LINKIS_VERSION.jar#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.spark.driver.conf.mainjar.*#wds.linkis.spark.driver.conf.mainjar=$SERVER_HOME/$SERVERNAME/conf:$SERVER_HOME/$SERVERNAME/lib/*#g\" $SERVER_CONF_PATH" +isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" +##SparkEM install end - -##pipelineEntrance install -PACKAGE_DIR=linkis/ujes/pipeline -SERVERNAME=linkis-ujes-pipeline-entrance -SERVER_PORT=$PEPELINE_ENTRANCE_PORT +##SparkEntrance install +PACKAGE_DIR=linkis/ujes/spark +SERVERNAME=linkis-ujes-spark-entrance +SERVER_PORT=$SPARK_ENTRANCE_PORT ###install dir installPackage ###update linkis.properties echo "$SERVERNAME-step4:update linkis conf" SERVER_CONF_PATH=$SERVER_HOME/$SERVERNAME/conf/linkis.properties -ssh $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$USER_LOG_PATH#g\" $SERVER_CONF_PATH" -ssh $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$RESULT_STORE_PATH#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.entrance.config.logPath.*#wds.linkis.entrance.config.logPath=$WORKSPACE_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" +ssh $SERVER_IP "sed -i \"s#wds.linkis.resultSet.store.path.*#wds.linkis.resultSet.store.path=$HDFS_USER_ROOT_PATH#g\" $SERVER_CONF_PATH" +isSuccess "subsitution linkis.properties of $SERVERNAME" echo "<----------------$SERVERNAME:end------------------->" - - - - - - - - -##init db -#mysql -h$MYSQL_HOST -p$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB -e "source ${workDir}/db/linkis_ddl.sql" -#mysql -h$MYSQL_HOST -p$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB -e "source ${workDir}/db/linkis_dml.sql" +##SparkEntrance install end \ No newline at end of file diff --git a/bin/start-all.sh b/bin/start-all.sh index 41d27e277bf..be5de2ba932 100644 --- a/bin/start-all.sh +++ b/bin/start-all.sh @@ -30,8 +30,16 @@ workDir=`cd "$workDir"; pwd` CONF_DIR="${workDir}"/../conf CONF_FILE=${CONF_DIR}/config.sh +function isSuccess(){ +if [ $? -ne 0 ]; then + echo "ERROR: " + $1 + exit 1 +else + echo "INFO:" + $1 +fi +} #安装dos2unix -sudo yum install dos2unix +sudo yum install dos2unix > /dev/null 2>&1 #获取本机IP local_host="`hostname --fqdn`" @@ -46,179 +54,205 @@ if [ -z ${LINKIS_INSTALL_HOME} ];then source ${CONF_FILE} fi fi - APP_PREFIX="linkis-" #顺序启动linkis的各个微服务,如果用户没有指定微服务的部署ip，则在本地启动 #eureka -echo "Eureka Server is Starting" +echo "<-------------------------------->" +echo "Begin to start Eureka Server" EUREKA_NAME="eureka" EUREKA_BIN=${LINKIS_INSTALL_HOME}/${EUREKA_NAME}/bin -EUREKA_START_CMD="cd ${EUREKA_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-${EUREKA_NAME}.sh > /dev/null" +EUREKA_START_CMD="cd ${EUREKA_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-${EUREKA_NAME}.sh > /dev/null" if [ -n "${EUREKA_INSTALL_IP}" ];then ssh ${EUREKA_INSTALL_IP} "${EUREKA_START_CMD}" + else ssh ${local_host} "${EUREKA_START_CMD}" fi -echo "Eureka Server started " - +isSuccess "End to start Eureka Server" +echo "<-------------------------------->" sleep 3 #gateway -echo "Gateway is Starting" +echo "<-------------------------------->" +echo "Begin to start Gateway" GATEWAY_NAME="gateway" GATEWAY_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${GATEWAY_NAME}/bin -GATEWAY_START_CMD="cd ${GATEWAY_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-${GATEWAY_NAME}.sh > /dev/null" +GATEWAY_START_CMD="cd ${GATEWAY_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-${GATEWAY_NAME}.sh > /dev/null" if [ -n "${GATEWAY_INSTALL_IP}" ];then ssh ${GATEWAY_INSTALL_IP} "${GATEWAY_START_CMD}" else ssh ${local_host} "${GATEWAY_START_CMD}" fi -echo "Gateway started " - +isSuccess "End to start Gateway" +echo "<-------------------------------->" sleep 3 #pub_service -echo "Public Service is Starting" +echo "<-------------------------------->" +echo "Begin to start Public Service" PUB_SERVICE_NAME="publicservice" PUB_SERVICE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PUB_SERVICE_NAME}/bin -PUB_SERVICE_START_CMD="cd ${PUB_SERVICE_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-${PUB_SERVICE_NAME}.sh > /dev/null" -if [ -n "${PUB_SERVICE_INSTALL_IP}" ];then - ssh ${PUB_SERVICE_INSTALL_IP} "${PUB_SERVICE_START_CMD}" +PUB_SERVICE_START_CMD="cd ${PUB_SERVICE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-${PUB_SERVICE_NAME}.sh > /dev/null" +if [ -n "${PUBLICSERVICE_INSTALL_IP}" ];then + ssh ${PUBLICSERVICE_INSTALL_IP} "${PUB_SERVICE_START_CMD}" else - ssh ${local_host} "${PUB_SERVICE_START_CMD}" + ssh ${local_host} "${PUB_SERVICE_START_CMD}" fi -echo "Public Service started " +isSuccess "End to start Public Service" +echo "<-------------------------------->" sleep 3 #database -echo "Database is Starting" +echo "<-------------------------------->" +echo "Begin to start Database" DATABASE_NAME="database" DATABASE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${DATABASE_NAME}/bin -DATABASE_START_CMD="cd ${DATABASE_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-${DATABASE_NAME}.sh > /dev/null" +DATABASE_START_CMD="if [ -d ${DATABASE_BIN} ];then cd ${DATABASE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-database.sh > /dev/null;else echo 'WARNING:Database will not start';fi" if [ -n "${DATABASE_INSTALL_IP}" ];then ssh ${DATABASE_INSTALL_IP} "${DATABASE_START_CMD}" else ssh ${local_host} "${DATABASE_START_CMD}" fi -echo "Database started " +isSuccess "End to start Database" +echo "<-------------------------------->" sleep 3 #Resource Manager -echo "Resource Manager is Starting" +echo "<-------------------------------->" +echo "Begin to start Resource Manager" RM_NAME="resourcemanager" RM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${RM_NAME}/bin -RM_START_CMD="cd ${RM_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-${RM_NAME}.sh > /dev/null" -if [ -n "${RM_INSTALL_IP}" ];then - ssh ${RM_INSTALL_IP} "${RM_START_CMD}" +RM_START_CMD="cd ${RM_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-${RM_NAME}.sh > /dev/null" +if [ -n "${RESOURCEMANAGER_INSTALL_IP}" ];then + ssh ${RESOURCEMANAGER_INSTALL_IP} "${RM_START_CMD}" else ssh ${local_host} "${RM_START_CMD}" fi -echo "Resource Manager started " - +isSuccess "End to start Resource Manager" +echo "<-------------------------------->" -echo "sleep 10 seconds to wait RM to start" -sleep 10 +echo "sleep 15 seconds to wait RM to be ready" +sleep 15 #SparkEntrance -echo "Spark Entrance is Starting" +echo "<-------------------------------->" +echo "Begin to start Spark Entrance" SPARK_ENTRANCE_NAME="ujes-spark-entrance" SPARK_ENTRANCE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${SPARK_ENTRANCE_NAME}/bin -SPARK_ENTRANCE_START_CMD="cd ${SPARK_ENTRANCE_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-sparkentrance.sh > /dev/null" +SPARK_ENTRANCE_START_CMD="if [ -d ${SPARK_ENTRANCE_BIN} ];then cd ${SPARK_ENTRANCE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-sparkentrance.sh > /dev/null;else echo 'WARNING:Spark Entrance will not start';fi" if [ -n "${SPARK_INSTALL_IP}" ];then ssh ${SPARK_INSTALL_IP} "${SPARK_ENTRANCE_START_CMD}" else ssh ${local_host} "${SPARK_ENTRANCE_START_CMD}" fi -echo "Spark Entrance started " - +echo "End to end Spark Entrance started" +echo "<-------------------------------->" +sleep 3 #Spark Engine Manager -echo "Spark Engine Manager is Starting" +echo "<-------------------------------->" +echo "Begin to Spark Engine Manager" SPARK_EM_NAME="ujes-spark-enginemanager" SPARK_EM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${SPARK_EM_NAME}/bin -SPARK_EM_START_CMD="cd ${SPARK_EM_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-sparkenginemanager.sh > /dev/null" +SPARK_EM_START_CMD="if [ -d ${SPARK_EM_BIN} ];then cd ${SPARK_EM_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-sparkenginemanager.sh > /dev/null;else echo 'WARNING:Spark EM will not start';fi" if [ -n "${SPARK_INSTALL_IP}" ];then ssh ${SPARK_INSTALL_IP} "${SPARK_EM_START_CMD}" else ssh ${local_host} "${SPARK_EM_START_CMD}" fi -echo "Spark Engine Manager started " - - +echo "End to start Spark Engine Manager " +echo "<-------------------------------->" +sleep 3 #HiveEntrance -echo "Hive Entrance is Starting" +echo "<-------------------------------->" +echo "Begin to start Hive Entrance" HIVE_ENTRANCE_NAME="ujes-hive-entrance" HIVE_ENTRANCE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${HIVE_ENTRANCE_NAME}/bin -HIVE_ENTRANCE_START_CMD="cd ${HIVE_ENTRANCE_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-hiveentrance.sh > /dev/null" +HIVE_ENTRANCE_START_CMD="if [ -d ${HIVE_ENTRANCE_BIN} ];then cd ${HIVE_ENTRANCE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-hiveentrance.sh > /dev/null;else echo 'WARNING:Hive Entrance will not start';fi" if [ -n "${HIVE_INSTALL_IP}" ];then ssh ${HIVE_INSTALL_IP} "${HIVE_ENTRANCE_START_CMD}" else ssh ${local_host} "${HIVE_ENTRANCE_START_CMD}" fi -echo "Hive Entrance started " +echo "End to start Hive Entrance" +echo "<-------------------------------->" + +sleep 3 + #Hive Engine Manager -echo "Hive Engine Manager is Starting" +echo "<-------------------------------->" +echo "Begin to start Hive Engine Manager" HIVE_EM_NAME="ujes-hive-enginemanager" HIVE_EM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${HIVE_EM_NAME}/bin -HIVE_EM_START_CMD="cd ${HIVE_EM_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-hiveenginemanager.sh > /dev/null" +HIVE_EM_START_CMD="if [ -d ${HIVE_EM_BIN} ];then cd ${HIVE_EM_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-hiveenginemanager.sh > /dev/null;else echo 'WARNING:Hive EM will not start';fi" if [ -n "${HIVE_INSTALL_IP}" ];then ssh ${HIVE_INSTALL_IP} "${HIVE_EM_START_CMD}" else ssh ${local_host} "${HIVE_EM_START_CMD}" fi -echo "Hive Engine Manager started " +echo "End to start Hive Engine Manager" +echo "<-------------------------------->" +sleep 3 #PythonEntrance -echo "Python Entrance is Starting" +echo "<-------------------------------->" +echo "Begin to start Python Entrance" PYTHON_ENTRANCE_NAME="ujes-python-entrance" PYTHON_ENTRANCE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PYTHON_ENTRANCE_NAME}/bin -PYTHON_ENTRANCE_START_CMD="cd ${PYTHON_ENTRANCE_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-pythonentrance.sh > /dev/null" +PYTHON_ENTRANCE_START_CMD="if [ -d ${PYTHON_ENTRANCE_BIN} ];then cd ${PYTHON_ENTRANCE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-pythonentrance.sh > /dev/null;else echo 'WARNING:Python Entrance will not start';fi" if [ -n "${PYTHON_INSTALL_IP}" ];then ssh ${PYTHON_INSTALL_IP} "${PYTHON_ENTRANCE_START_CMD}" else ssh ${local_host} "${PYTHON_ENTRANCE_START_CMD}" fi -echo "Python Entrance started " +echo "End to start Python Entrance" +echo "<-------------------------------->" + +sleep 3 #Python Engine Manager -echo "Python Engine Manager is Starting" +echo "<-------------------------------->" +echo "Begin to start Python Engine Manager" PYTHON_EM_NAME="ujes-python-enginemanager" PYTHON_EM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PYTHON_EM_NAME}/bin -PYTHON_EM_START_CMD="cd ${PYTHON_EM_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-pythonenginemanager.sh > /dev/null" +PYTHON_EM_START_CMD="if [ -d ${PYTHON_EM_BIN} ];then cd ${PYTHON_EM_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-pythonenginemanager.sh > /dev/null;else echo 'WARNING:Python EM will not start';fi" if [ -n "${PYTHON_INSTALL_IP}" ];then ssh ${PYTHON_INSTALL_IP} "${PYTHON_EM_START_CMD}" else ssh ${local_host} "${PYTHON_EM_START_CMD}" fi -echo "Python Engine Manager started " - - +echo "End to start Python Engine Manager" +echo "<-------------------------------->" +sleep 3 -#PipelineEntrance -echo "Pipeline Entrance is Starting" -PIPELINE_ENTRANCE_NAME="ujes-pipeline-entrance" -PIPELINE_ENTRANCE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PIPELINE_ENTRANCE_NAME}/bin -PIPELINE_ENTRANCE_START_CMD="cd ${PIPELINE_ENTRANCE_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-pipelineentrance.sh > /dev/null" -if [ -n "${PIPELINE_INSTALL_IP}" ];then - ssh ${PIPELINE_INSTALL_IP} "${PIPELINE_ENTRANCE_START_CMD}" -else - ssh ${local_host} "${PIPELINE_ENTRANCE_START_CMD}" -fi -echo "Pipeline Entrance started " - -#Pipeline Engine Manager -echo "Pipeline Engine Manager is Starting" -PIPELINE_EM_NAME="ujes-pipeline-enginemanager" -PIPELINE_EM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PIPELINE_EM_NAME}/bin -PIPELINE_EM_START_CMD="cd ${PIPELINE_EM_BIN}; dos2unix ./*; dos2unix ../conf/*; sh start-pipelineenginemanager.sh > /dev/null" -if [ -n "${PIPELINE_INSTALL_IP}" ];then - ssh ${PIPELINE_INSTALL_IP} "${PIPELINE_EM_START_CMD}" -else - ssh ${local_host} "${PIPELINE_EM_START_CMD}" -fi -echo "Pipeline Engine Manager started " \ No newline at end of file +##PipelineEntrance +#echo "Pipeline Entrance is Starting" +#PIPELINE_ENTRANCE_NAME="ujes-pipeline-entrance" +#PIPELINE_ENTRANCE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PIPELINE_ENTRANCE_NAME}/bin +#PIPELINE_ENTRANCE_START_CMD="cd ${PIPELINE_ENTRANCE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-pipelineentrance.sh > /dev/null" +#if [ -n "${PIPELINE_INSTALL_IP}" ];then +# ssh ${PIPELINE_INSTALL_IP} "${PIPELINE_ENTRANCE_START_CMD}" +#else +# ssh ${local_host} "${PIPELINE_ENTRANCE_START_CMD}" +#fi +#echo "Pipeline Entrance started " +# +##Pipeline Engine Manager +#echo "Pipeline Engine Manager is Starting" +#PIPELINE_EM_NAME="ujes-pipeline-enginemanager" +#PIPELINE_EM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PIPELINE_EM_NAME}/bin +#PIPELINE_EM_START_CMD="cd ${PIPELINE_EM_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh start-pipelineenginemanager.sh > /dev/null" +#if [ -n "${PIPELINE_INSTALL_IP}" ];then +# ssh ${PIPELINE_INSTALL_IP} "${PIPELINE_EM_START_CMD}" +#else +# ssh ${local_host} "${PIPELINE_EM_START_CMD}" +#fi +#echo "Pipeline Engine Manager started " + + +echo "start-all shell script executed completely" \ No newline at end of file diff --git a/bin/stop-all.sh b/bin/stop-all.sh index 7e8081c0e91..e49635c367a 100644 --- a/bin/stop-all.sh +++ b/bin/stop-all.sh @@ -30,8 +30,16 @@ workDir=`cd "$workDir"; pwd` CONF_DIR="${workDir}"/../conf CONF_FILE=${CONF_DIR}/config.sh +function isSuccess(){ +if [ $? -ne 0 ]; then + echo "ERROR: " + $1 + exit 1 +else + echo "INFO:" + $1 +fi +} #安装dos2unix -sudo yum install dos2unix +sudo yum install dos2unix > /dev/null 2>&1 #获取本机IP local_host="`hostname --fqdn`" @@ -46,172 +54,192 @@ if [ -z ${LINKIS_INSTALL_HOME} ];then source ${CONF_FILE} fi fi - APP_PREFIX="linkis-" #顺序启动linkis的各个微服务,如果用户没有指定微服务的部署ip，则在本地启动 #eureka -echo "Eureka Server is Stoping" +echo "<-------------------------------->" +echo "Begin to stop Eureka Server" EUREKA_NAME="eureka" EUREKA_BIN=${LINKIS_INSTALL_HOME}/${EUREKA_NAME}/bin -EUREKA_STOP_CMD="cd ${EUREKA_BIN}; sh stop-${EUREKA_NAME}.sh > /dev/null" +EUREKA_STOP_CMD="cd ${EUREKA_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-${EUREKA_NAME}.sh > /dev/null" if [ -n "${EUREKA_INSTALL_IP}" ];then ssh ${EUREKA_INSTALL_IP} "${EUREKA_STOP_CMD}" + else ssh ${local_host} "${EUREKA_STOP_CMD}" fi -echo "Eureka Server stoped " - +isSuccess "End to stop Eureka Server" +echo "<-------------------------------->" +sleep 3 #gateway -echo "Gateway is Stoping" +echo "<-------------------------------->" +echo "Begin to stop Gateway" GATEWAY_NAME="gateway" GATEWAY_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${GATEWAY_NAME}/bin -GATEWAY_STOP_CMD="cd ${GATEWAY_BIN}; sh stop-${GATEWAY_NAME}.sh > /dev/null" +GATEWAY_STOP_CMD="cd ${GATEWAY_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-${GATEWAY_NAME}.sh > /dev/null" if [ -n "${GATEWAY_INSTALL_IP}" ];then ssh ${GATEWAY_INSTALL_IP} "${GATEWAY_STOP_CMD}" else ssh ${local_host} "${GATEWAY_STOP_CMD}" fi -echo "Gateway stoped " +isSuccess "End to stop Gateway" +echo "<-------------------------------->" +sleep 3 #pub_service -echo "Public Service is Stoping" +echo "<-------------------------------->" +echo "Begin to stop Public Service" PUB_SERVICE_NAME="publicservice" PUB_SERVICE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PUB_SERVICE_NAME}/bin -PUB_SERVICE_STOP_CMD="cd ${PUB_SERVICE_BIN}; sh stop-${PUB_SERVICE_NAME}.sh > /dev/null" -if [ -n "${PUB_SERVICE_INSTALL_IP}" ];then - ssh ${PUB_SERVICE_INSTALL_IP} "${PUB_SERVICE_STOP_CMD}" +PUB_SERVICE_STOP_CMD="cd ${PUB_SERVICE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-${PUB_SERVICE_NAME}.sh > /dev/null" +if [ -n "${PUBLICSERVICE_INSTALL_IP}" ];then + ssh ${PUBLICSERVICE_INSTALL_IP} "${PUB_SERVICE_STOP_CMD}" else - ssh ${local_host} "${PUB_SERVICE_STOP_CMD}" + ssh ${local_host} "${PUB_SERVICE_STOP_CMD}" fi -echo "Public Service stoped " - +isSuccess "End to stop Public Service" +echo "<-------------------------------->" +sleep 3 #database -echo "Database is Stoping" +echo "<-------------------------------->" +echo "Begin to stop Database" DATABASE_NAME="database" DATABASE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${DATABASE_NAME}/bin -DATABASE_STOP_CMD="cd ${DATABASE_BIN}; sh stop-${DATABASE_NAME}.sh > /dev/null" +DATABASE_STOP_CMD="if [ -d ${DATABASE_BIN} ];then cd ${DATABASE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-database.sh > /dev/null;else echo 'WARNING:Database did not start';fi" if [ -n "${DATABASE_INSTALL_IP}" ];then ssh ${DATABASE_INSTALL_IP} "${DATABASE_STOP_CMD}" else ssh ${local_host} "${DATABASE_STOP_CMD}" fi -echo "database stoped " - +isSuccess "End to stop Database" +echo "<-------------------------------->" +sleep 3 #Resource Manager -echo "Resource Manager is Stoping" +echo "<-------------------------------->" +echo "Begin to stop Resource Manager" RM_NAME="resourcemanager" RM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${RM_NAME}/bin -RM_STOP_CMD="cd ${RM_BIN}; sh stop-${RM_NAME}.sh > /dev/null" -if [ -n "${RM_INSTALL_IP}" ];then - ssh ${RM_INSTALL_IP} "${RM_STOP_CMD}" +RM_STOP_CMD="cd ${RM_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-${RM_NAME}.sh > /dev/null" +if [ -n "${RESOURCEMANAGER_INSTALL_IP}" ];then + ssh ${RESOURCEMANAGER_INSTALL_IP} "${RM_STOP_CMD}" else ssh ${local_host} "${RM_STOP_CMD}" fi -echo "Resource Manager stoped " +isSuccess "End to stop Resource Manager" +echo "<-------------------------------->" + +sleep 3 #SparkEntrance -echo "Spark Entrance is Stoping" +echo "<-------------------------------->" +echo "Begin to stop Spark Entrance" SPARK_ENTRANCE_NAME="ujes-spark-entrance" SPARK_ENTRANCE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${SPARK_ENTRANCE_NAME}/bin -SPARK_ENTRANCE_STOP_CMD="cd ${SPARK_ENTRANCE_BIN}; sh stop-sparkentrance.sh > /dev/null" +SPARK_ENTRANCE_STOP_CMD="if [ -d ${SPARK_ENTRANCE_BIN} ];then cd ${SPARK_ENTRANCE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-sparkentrance.sh > /dev/null;else echo 'WARNING:Spark Entrance did not start';fi" if [ -n "${SPARK_INSTALL_IP}" ];then ssh ${SPARK_INSTALL_IP} "${SPARK_ENTRANCE_STOP_CMD}" else ssh ${local_host} "${SPARK_ENTRANCE_STOP_CMD}" fi -echo "Spark Entrance stoped " +echo "End to stop Spark Entrance" +echo "<-------------------------------->" #Spark Engine Manager -echo "Spark Engine Manager is Stoping" +echo "<-------------------------------->" +echo "Begin to Spark Engine Manager" SPARK_EM_NAME="ujes-spark-enginemanager" SPARK_EM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${SPARK_EM_NAME}/bin -SPARK_EM_STOP_CMD="cd ${SPARK_EM_BIN}; sh stop-sparkenginemanager.sh > /dev/null" +SPARK_EM_STOP_CMD="if [ -d ${SPARK_EM_BIN} ];then cd ${SPARK_EM_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-sparkenginemanager.sh > /dev/null;else echo 'WARNING:Spark EM did not start';fi" if [ -n "${SPARK_INSTALL_IP}" ];then ssh ${SPARK_INSTALL_IP} "${SPARK_EM_STOP_CMD}" else ssh ${local_host} "${SPARK_EM_STOP_CMD}" fi -echo "Spark Engine Manager stoped " - +echo "End to stop Spark Engine Manager " +echo "<-------------------------------->" #HiveEntrance -echo "Hive Entrance is Stoping" +echo "<-------------------------------->" +echo "Begin to stop Hive Entrance" HIVE_ENTRANCE_NAME="ujes-hive-entrance" HIVE_ENTRANCE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${HIVE_ENTRANCE_NAME}/bin -HIVE_ENTRANCE_STOP_CMD="cd ${HIVE_ENTRANCE_BIN}; sh stop-hiveentrance.sh > /dev/null" +HIVE_ENTRANCE_STOP_CMD="if [ -d ${HIVE_ENTRANCE_BIN} ];then cd ${HIVE_ENTRANCE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-hiveentrance.sh > /dev/null;else echo 'WARNING:Hive Entrance did not start';fi" if [ -n "${HIVE_INSTALL_IP}" ];then ssh ${HIVE_INSTALL_IP} "${HIVE_ENTRANCE_STOP_CMD}" else ssh ${local_host} "${HIVE_ENTRANCE_STOP_CMD}" fi -echo "Hive Entrance stoped " - +echo "End to stop Hive Entrance" +echo "<-------------------------------->" #Hive Engine Manager -echo "Hive Engine Manager is Stoping" +echo "<-------------------------------->" +echo "Begin to stop Hive Engine Manager" HIVE_EM_NAME="ujes-hive-enginemanager" HIVE_EM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${HIVE_EM_NAME}/bin -HIVE_EM_STOP_CMD="cd ${HIVE_EM_BIN}; sh stop-hiveenginemanager.sh > /dev/null" +HIVE_EM_STOP_CMD="if [ -d ${HIVE_EM_BIN} ];then cd ${HIVE_EM_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-hiveenginemanager.sh > /dev/null;else echo 'WARNING:Hive EM did not start';fi" if [ -n "${HIVE_INSTALL_IP}" ];then ssh ${HIVE_INSTALL_IP} "${HIVE_EM_STOP_CMD}" else ssh ${local_host} "${HIVE_EM_STOP_CMD}" fi -echo "Hive Engine Manager stoped " - +echo "End to stop Hive Engine Manager" +echo "<-------------------------------->" #PythonEntrance -echo "Python Entrance is Stoping" +echo "<-------------------------------->" +echo "Begin to stop Python Entrance" PYTHON_ENTRANCE_NAME="ujes-python-entrance" PYTHON_ENTRANCE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PYTHON_ENTRANCE_NAME}/bin -PYTHON_ENTRANCE_STOP_CMD="cd ${PYTHON_ENTRANCE_BIN}; sh stop-pythonentrance.sh > /dev/null" +PYTHON_ENTRANCE_STOP_CMD="if [ -d ${PYTHON_ENTRANCE_BIN} ];then cd ${PYTHON_ENTRANCE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-pythonentrance.sh > /dev/null;else echo 'WARNING:Python Entrance did not start';fi" if [ -n "${PYTHON_INSTALL_IP}" ];then ssh ${PYTHON_INSTALL_IP} "${PYTHON_ENTRANCE_STOP_CMD}" else ssh ${local_host} "${PYTHON_ENTRANCE_STOP_CMD}" fi -echo "Python Entrance stoped " - +echo "End to stop Python Entrance" +echo "<-------------------------------->" #Python Engine Manager -echo "Python Engine Manager is Stoping" +echo "<-------------------------------->" +echo "Begin to stop Python Engine Manager" PYTHON_EM_NAME="ujes-python-enginemanager" PYTHON_EM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PYTHON_EM_NAME}/bin -PYTHON_EM_STOP_CMD="cd ${PYTHON_EM_BIN}; sh stop-pythonenginemanager.sh > /dev/null" +PYTHON_EM_STOP_CMD="if [ -d ${PYTHON_EM_BIN} ];then cd ${PYTHON_EM_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-pythonenginemanager.sh > /dev/null;else echo 'WARNING:Python EM did not start';fi" if [ -n "${PYTHON_INSTALL_IP}" ];then ssh ${PYTHON_INSTALL_IP} "${PYTHON_EM_STOP_CMD}" else ssh ${local_host} "${PYTHON_EM_STOP_CMD}" fi -echo "Python Engine Manager stoped " - - - - -#PipelineEntrance -echo "Pipeline Entrance is Stoping" -PIPELINE_ENTRANCE_NAME="ujes-pipeline-entrance" -PIPELINE_ENTRANCE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PIPELINE_ENTRANCE_NAME}/bin -PIPELINE_ENTRANCE_STOP_CMD="cd ${PIPELINE_ENTRANCE_BIN}; sh stop-pipelineentrance.sh > /dev/null" -if [ -n "${PIPELINE_INSTALL_IP}" ];then - ssh ${PIPELINE_INSTALL_IP} "${PIPELINE_ENTRANCE_STOP_CMD}" -else - ssh ${local_host} "${PIPELINE_ENTRANCE_STOP_CMD}" -fi -echo "Pipeline Entrance stoped " - -#Pipeline Engine Manager -echo "Pipeline Engine Manager is Stoping" -PIPELINE_EM_NAME="ujes-pipeline-enginemanager" -PIPELINE_EM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PIPELINE_EM_NAME}/bin -PIPELINE_EM_STOP_CMD="cd ${PIPELINE_EM_BIN}; sh stop-pipelineenginemanager.sh > /dev/null" -if [ -n "${PIPELINE_INSTALL_IP}" ];then - ssh ${PIPELINE_INSTALL_IP} "${PIPELINE_EM_STOP_CMD}" -else - ssh ${local_host} "${PIPELINE_EM_STOP_CMD}" -fi -echo "Pipeline Engine Manager stoped " \ No newline at end of file +echo "End to stop Python Engine Manager" +echo "<-------------------------------->" + + + +##PipelineEntrance +#echo "Pipeline Entrance is Stoping" +#PIPELINE_ENTRANCE_NAME="ujes-pipeline-entrance" +#PIPELINE_ENTRANCE_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PIPELINE_ENTRANCE_NAME}/bin +#PIPELINE_ENTRANCE_STOP_CMD="cd ${PIPELINE_ENTRANCE_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-pipelineentrance.sh > /dev/null" +#if [ -n "${PIPELINE_INSTALL_IP}" ];then +# ssh ${PIPELINE_INSTALL_IP} "${PIPELINE_ENTRANCE_STOP_CMD}" +#else +# ssh ${local_host} "${PIPELINE_ENTRANCE_STOP_CMD}" +#fi +#echo "Pipeline Entrance stoped " +# +##Pipeline Engine Manager +#echo "Pipeline Engine Manager is Stoping" +#PIPELINE_EM_NAME="ujes-pipeline-enginemanager" +#PIPELINE_EM_BIN=${LINKIS_INSTALL_HOME}/${APP_PREFIX}${PIPELINE_EM_NAME}/bin +#PIPELINE_EM_STOP_CMD="cd ${PIPELINE_EM_BIN}; dos2unix ./* > /dev/null 2>&1; dos2unix ../conf/* > /dev/null 2>&1; sh stop-pipelineenginemanager.sh > /dev/null" +#if [ -n "${PIPELINE_INSTALL_IP}" ];then +# ssh ${PIPELINE_INSTALL_IP} "${PIPELINE_EM_STOP_CMD}" +#else +# ssh ${local_host} "${PIPELINE_EM_STOP_CMD}" +#fi +#echo "Pipeline Engine Manager stoped " \ No newline at end of file diff --git a/conf/config.sh b/conf/config.sh index 72d8195e541..808ecf0928d 100644 --- a/conf/config.sh +++ b/conf/config.sh @@ -1,56 +1,87 @@ -#user to deploy +### deploy user deployUser=hadoop -#sample /appcom/Install/WillinkInstall -LINKIS_INSTALL_HOME= -#linkis version -LINKIS_VERSION=0.5.0 -#path to store user's log -USER_LOG_PATH=file:// -#path to store result -RESULT_STORE_PATH=hdfs:// -##EUREKA -EUREKA_INSTALL_IP= + + +### The install home path of Linkis +LINKIS_INSTALL_HOME=/appcom/Install/Linkis #Must provided + + + +### Specifies the user workspace, which is used to store the user's script files and log files. +### Generally local directory +WORKSPACE_USER_ROOT_PATH=file:///tmp/linkis/ +### User's root hdfs path +HDFS_USER_ROOT_PATH=hdfs:///tmp/linkis + +### Path to store job ResultSet：file or hdfs path +RESULT_SET_ROOT_PATH=hdfs:///tmp/linkis + +### Provide the DB information of Hive metadata database. +#HIVE_META_URL= +#HIVE_META_USER= +#HIVE_META_PASSWORD= + + +################### The install Configuration of all Micro-Services ##################### +# +# NOTICE: +# 1. If you just wanna try, the following micro-service configuration can be set without any settings. +# These services will be installed by default on this machine. +# 2. In order to get the most complete enterprise-level features, we strongly recommend that you install +# Linkis in a distributed manner and set the following microservice parameters +# + +### EUREKA install information。 +### You can access it in your browser at the address below：http://${EUREKA_INSTALL_IP}:${EUREKA_PORT} +EUREKA_INSTALL_IP=127.0.0.1 # Microservices Service Registration Discovery Center EUREKA_PORT=20303 -##Gateway -GATEWAY_INSTALL_IP= +### Gateway install information +GATEWAY_INSTALL_IP=127.0.0.1 GATEWAY_PORT=9001 -LDAP_URL=ldap://localhost:1389/ -LDAP_BASEDN=dc=webank,dc=com -##publicservice -#PUBLICSERVICE_INSTALL_IP +### publicservice +PUBLICSERVICE_INSTALL_IP=127.0.0.1 PUBLICSERVICE_PORT=9002 -WORKSPACE_PATH=file:///tmp/linkis/ -HDFS_PATH=hdfs:///tmp/linkis -#HIVE_META_URL -#HIVE_META_USER -#HIVE_META_PASSWORD -##RM -#RESOURCEMANAGER_INSTALL_IP +### Hive Metadata Query service, provide the metadata information of Hive databases. +DATABASE_INSTALL_IP=127.0.0.1 +DATABASE_PORT=9008 + + +### ResourceManager +RESOURCEMANAGER_INSTALL_IP=127.0.0.1 RESOURCEMANAGER_PORT=9003 -##Spark -#SPARK_INSTALL_IP + +### Spark +### This service is used to provide spark capability. +SPARK_INSTALL_IP=127.0.0.1 SPARK_EM_PORT=10001 SPARK_ENTRANCE_PORT=10002 -##Hive -#HIVE_INSTALL_IP + +### Hive +### This service is used to provide hive capability. +HIVE_INSTALL_IP=127.0.0.1 HIVE_EM_PORT=11001 HIVE_ENTRANCE_PORT=11002 -##PYTHON -#PYTHON_INSTALL_IP +### PYTHON +### This service is used to provide python capability. +PYTHON_INSTALL_IP=127.0.0.1 PYTHON_EM_PORT=12001 PYTHON_ENTRANCE_PORT=12002 -##COMMON CONFIG -HADOOP_CONF_DIR=/appcom/config/hadoop-config -SPARK_CONF_DIR= -HADOOP_HOME= -SPARK_HOME= \ No newline at end of file + +######################################################################################## + +## LDAP is for enterprise authorization, if you just want to have a try, ignore it. +#LDAP_URL=ldap://localhost:1389/ +#LDAP_BASEDN=dc=webank,dc=com + + +LINKIS_VERSION=0.8.0 \ No newline at end of file diff --git a/conf/db.properties b/conf/db.properties deleted file mode 100644 index 903316ae042..00000000000 --- a/conf/db.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - diff --git a/conf/db.sh b/conf/db.sh index 47ab125c2f2..f19e83d7394 100644 --- a/conf/db.sh +++ b/conf/db.sh @@ -1,3 +1,4 @@ +### Used to store user's custom variables, user's configuration, UDFs and functions, while providing the JobHistory service MYSQL_HOST= MYSQL_PORT= MYSQL_DB= diff --git a/conf/install.env b/conf/install.env deleted file mode 100644 index d6271a15524..00000000000 --- a/conf/install.env +++ /dev/null @@ -1,10 +0,0 @@ -# env配置,其中LINKIS_INSTALL_HOME为必须配置选择,IP则是选配，如果不配置IP,则默认在本地 -LINKIS_INSTALL_HOME=/appcom/Install/DataWorkCloudInstall -#EUREKA_INSTALL_IP= -#GATEWAY_INSTALL_IP= -#PUB_SERVICE_INSTALL_IP= -#RM_INSTALL_IP= -#HIVE_INSTALL_IP=locahost -#SPARK_INSTALL_IP= -#PYTHON_INSTALL_IP= -#TIDB_INSTALL_IP= \ No newline at end of file diff --git a/conf/install.properties b/conf/install.properties deleted file mode 100644 index 903316ae042..00000000000 --- a/conf/install.properties +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright 2019 WeBank -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - diff --git a/core/cloudModule/pom.xml b/core/cloudModule/pom.xml index c5a68e029ad..14fde1bb530 100644 --- a/core/cloudModule/pom.xml +++ b/core/cloudModule/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 4.0.0 @@ -286,6 +286,22 @@ com.google.code.gson gson + + io.netty + netty-all + ${netty.version} + + + org.json4s + json4s-jackson_${scala.binary.version} + ${json4s.version} + + + org.scala-lang + scala-library + + + diff --git a/core/cloudModule/src/main/assembly/distribution.xml b/core/cloudModule/src/main/assembly/distribution.xml index 77b2f3acbef..1f2c5e0a6aa 100644 --- a/core/cloudModule/src/main/assembly/distribution.xml +++ b/core/cloudModule/src/main/assembly/distribution.xml @@ -35,6 +35,14 @@ false true true + + io.netty:netty-buffer* + io.netty:netty-codec* + io.netty:netty-common* + io.netty:netty-handler* + io.netty:netty-transport* + io.netty:netty:jar + diff --git a/core/cloudMybatis/pom.xml b/core/cloudMybatis/pom.xml index 786f1115fc4..470955b5e0b 100644 --- a/core/cloudMybatis/pom.xml +++ b/core/cloudMybatis/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 4.0.0 jar diff --git a/core/cloudProtocol/pom.xml b/core/cloudProtocol/pom.xml index 93d050568a1..5c74aa31777 100644 --- a/core/cloudProtocol/pom.xml +++ b/core/cloudProtocol/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 4.0.0 diff --git a/core/cloudRPC/pom.xml b/core/cloudRPC/pom.xml index 8e7d563d63f..6ae6c7863f4 100644 --- a/core/cloudRPC/pom.xml +++ b/core/cloudRPC/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 4.0.0 diff --git a/core/common/pom.xml b/core/common/pom.xml index 73e9bc69142..684ead08e02 100644 --- a/core/common/pom.xml +++ b/core/common/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 4.0.0 diff --git a/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/BDPConfiguration.scala b/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/BDPConfiguration.scala index 4f121be87ce..8e440839060 100644 --- a/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/BDPConfiguration.scala +++ b/core/common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/BDPConfiguration.scala @@ -36,6 +36,8 @@ private[conf] object BDPConfiguration extends Logging { private val sysProps = sys.props private val extractConfig = new Properties + private val env = sys.env + val propertyFile = sysProps.getOrElse("wds.linkis.configuration", DEFAULT_PROPERTY_FILE_NAME) private val configFileURL = getClass.getClassLoader.getResource(propertyFile) info(configFileURL.getPath) @@ -59,7 +61,11 @@ private[conf] object BDPConfiguration extends Logging { if(StringUtils.isNotEmpty(value)) { return Some(value) } - sysProps.get(key).orElse(sys.props.get(key)) + val propsValue = sysProps.get(key).orElse(sys.props.get(key)) + if(propsValue.isDefined){ + return propsValue + } + env.get(key) } def properties = { @@ -67,6 +73,7 @@ private[conf] object BDPConfiguration extends Logging { props.putAll(sysProps) props.putAll(config) props.putAll(extractConfig) + props.putAll(env) props } diff --git a/core/httpclient/pom.xml b/core/httpclient/pom.xml index fe373b2a360..bebe1e3e173 100644 --- a/core/httpclient/pom.xml +++ b/core/httpclient/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 4.0.0 diff --git a/core/scheduler/pom.xml b/core/scheduler/pom.xml index f507351d954..3103a063471 100644 --- a/core/scheduler/pom.xml +++ b/core/scheduler/pom.xml @@ -22,7 +22,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-scheduler @@ -66,7 +66,7 @@ product - 0.0.3-SNAPSHOT + 0.0.5 diff --git a/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/UserJob.scala b/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/UserJob.scala index 3dd5b9cff2e..678253e1679 100644 --- a/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/UserJob.scala +++ b/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/UserJob.scala @@ -20,13 +20,13 @@ package com.webank.wedatasphere.linkis.scheduler.queue * Created by enjoyyin on 2018/9/12. */ case class UserJob() extends Job{ - override def init() = ??? + override def init() = {} - override protected def jobToExecuteRequest = ??? + override protected def jobToExecuteRequest = null - override def getName = ??? + override def getName = null - override def getJobInfo = ??? + override def getJobInfo = null - override def close() = ??? + override def close() = {} } \ No newline at end of file diff --git a/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/fifoqueue/FIFOSchedulerContextImpl.scala b/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/fifoqueue/FIFOSchedulerContextImpl.scala index 5559e9ee110..7839ffdf756 100644 --- a/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/fifoqueue/FIFOSchedulerContextImpl.scala +++ b/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/fifoqueue/FIFOSchedulerContextImpl.scala @@ -46,7 +46,7 @@ class FIFOSchedulerContextImpl(val maxParallelismUsers: Int) extends SchedulerC } } - override def getOrCreateExecutorManager = ??? + override def getOrCreateExecutorManager = null - override def getOrCreateSchedulerListenerBus = ??? + override def getOrCreateSchedulerListenerBus = null } diff --git a/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/parallelqueue/ParallelSchedulerContextImpl.scala b/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/parallelqueue/ParallelSchedulerContextImpl.scala index c3a5f143b09..83e2086fbb8 100644 --- a/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/parallelqueue/ParallelSchedulerContextImpl.scala +++ b/core/scheduler/src/main/scala/com/webank/wedatasphere/linkis/scheduler/queue/parallelqueue/ParallelSchedulerContextImpl.scala @@ -48,7 +48,7 @@ class ParallelSchedulerContextImpl(val maxParallelismUsers: Int) extends Schedu } } - override def getOrCreateExecutorManager = ??? + override def getOrCreateExecutorManager = null - override def getOrCreateSchedulerListenerBus = ??? + override def getOrCreateSchedulerListenerBus = null } diff --git a/db/linkis_ddl.sql b/db/linkis_ddl.sql index 934d7d7b14c..6be8c8dd495 100644 --- a/db/linkis_ddl.sql +++ b/db/linkis_ddl.sql @@ -77,7 +77,7 @@ CREATE TABLE `linkis_user` ( `avatar` varchar(255) DEFAULT NULL COMMENT 'Path of the avator', `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `create_by` bigint(20) DEFAULT '0', - `update_time` timestamp NOT NULL DEFAULT '1970-01-01 08:00:01', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `update_by` bigint(20) DEFAULT '0', `is_first_login` bit(1) DEFAULT NULL COMMENT 'If it is the first time to log in', PRIMARY KEY (`id`) @@ -291,11 +291,11 @@ DROP TABLE IF EXISTS `linkis_udf_tree`; CREATE TABLE `linkis_udf_tree` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, `parent` bigint(20) NOT NULL, - `name` varchar(100) DEFAULT NULL COMMENT 'Category name of the function. IT would be displayed in the front-end', + `name` varchar(100) DEFAULT NULL COMMENT 'Category name of the function. It would be displayed in the front-end', `user_name` varchar(50) NOT NULL, `description` varchar(255) DEFAULT NULL, `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `update_time` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, `category` varchar(50) DEFAULT NULL COMMENT 'Used to distinguish between udf and function', PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; @@ -329,7 +329,7 @@ CREATE TABLE `linkis_udf` ( `is_shared` bit(1) DEFAULT NULL, `tree_id` bigint(20) NOT NULL, `create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `update_time` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/docs/en_US/SUMMARY.md b/docs/en_US/SUMMARY.md new file mode 100644 index 00000000000..93de955d748 --- /dev/null +++ b/docs/en_US/SUMMARY.md @@ -0,0 +1,9 @@ +# Summary +* [Introduction](README.md) +* [System deployment documentation](ch1/deploy.md) +* [linkis quick start](ch2/Linkis%20Quick%20Start.md) +* [Linkis User Manual](ch3/Linkis%20User%20Manual.md) + + +​ + diff --git a/docs/en_US/ch1/deploy.md b/docs/en_US/ch1/deploy.md new file mode 100644 index 00000000000..dabf58ad784 --- /dev/null +++ b/docs/en_US/ch1/deploy.md @@ -0,0 +1,98 @@ +## 1 Getting Started + +### (1) Prerequisites(Please install the items marked with "required") + +- Mysql (5.5+) : Required +- JDK (1.8.0_141) : Required +- Hadoop (2.7.2) : Required +- Hive (1.2.1) : Optional, Hive engine nodes need to be installed. +- Spark (2.1.0) : Optional, Spark engine nodes need to be installed. + +Notes: Linkis itself does not depend on Hadoop, it only calls their clients when running the corresponding tasks. + +### (2) Create deploy users + +Create deploy users on all machines that needs to be deployed. Since the deploy service executes the job in sudo -u {linux-user} mode, the deploy user needs to have sudo privileges, and it needs to be passwordless. + +``` +vi /etc/sudoers +``` + +For example, a deploy user is a hadoop account. + +``` +hadoop ALL=(ALL) NOPASSWD: NOPASSWD: ALL +``` + +### (3) ssh passwordless configuration + +Setup passwordless SSH login on deploy machines and other installation machines. If you want to install on deploy machine, you need to get through the ssh of the host machine and other machines. + +## 2 Compile and Package + +After getting the project from git, use maven to package the project installation package(If you already have the installation package, install it directly, no need to compile). + +(1) Execute the following command in the directory where the outermost project pom.xml locates. + +``` + mvn clean install +``` + +(2) Execute the following command if it is the first time use on local server, otherwise just execute the command in step 1: + +``` + mvn -N install +``` + +(3) Get the installation package, in the assembly->target directory: + +``` + wedatasphere-linkis-x.x.x-dist.tar.gz +``` + +## 3 Installation + +Unzip the installation package to the installation directory and modify the configuration file after decompression. + +``` + tar -xvf wedatasphere-linkis-x.x.x-dist.tar.gz +``` + +(1) Modify the basic configuration + +``` + vi /conf/config.sh +``` + +- Specify deploy user: deployUser +- Specify the installation directory: LINKIS_INSTALL_HOME +- Specify the log storage path: USER_LOG_PATH +- Specify the result set HDFS storage path: RESULT_STORE_PATH +- Specify the machine IP address and port number where each service is installed: * _INSTALL_IP, * _PORT + +(2) Modify the configuration of database + +``` + vi ./conf/db.sh +``` + +- Set the connection information of the database +- Including IP address, database name, username, port +- MYSQL_HOST= +- MYSQL_PORT= +- MYSQL_DB= +- MYSQL_USER= +- MYSQL_PASSWORD= + +(3) Execute the installation script + +``` + sh ./bin/install.sh +``` + +## 4 Start Service + +``` + sh ./bin/start-all.sh +``` + diff --git a/docs/en_US/ch2/Linkis Quick Start.md b/docs/en_US/ch2/Linkis Quick Start.md new file mode 100644 index 00000000000..0aee9d59047 --- /dev/null +++ b/docs/en_US/ch2/Linkis Quick Start.md @@ -0,0 +1,127 @@ +### Linkis Quick Start + +#### 1.Summary +Linkis provides a client implementation for users to have quick-access to Linkis background services using UJESClient. +#### 2.How to Use UJESClient +**2.1 Client Configuration** + +Configurations are necessary before instantiating a UJESClient object. To do the configuration, an instance of DWSClientConfig should be obtained from DWSClientBuilder. Parameters should be specified as below: +- ServerUrl: The address of Linkis gateway, i.e. http://{ip}:{port} +- connectionTimeOut: The connection timeout of the client +- AuthenticationStrategy: The authentication strategy of Linkis +- AuthTokenKey: The authentication key，usually the username +- AuthTokenValue: The authentication value，usually the password +- DWSVersion: The version of Linkis background protocol, currently v1 + +**2.2 UJESClient Instantiation** + +The instantiation of UJESClient is very convenient. Simply passing a ClientConfig object configured as mentioned in 2.1 can easily instantiate a new object. + +**2.3 Service Request by UJESClient** + +After the instantiation of UJESClient, the 'execute' method is invokable to request the background services of Linkis. An instance of JobExectutionAction should be passed as parameter, which can be obtained by invoking JobExecutionAction.builder() with following parameters: +- creator: The name of the system which holds the UJES client and requests for Linkis +- EngineType: The engine type expected by the client, i.e. Spark, Hive, etc... +- User: The user who makes this request +- ExecutionCode: The code which is requested to be executed +After the UJESClient submitted the JobExecutionAction instance to Linkis services, a JobExecutionResult with execId and taskID information would be returned. To track the logs or the status of a task, users can pass the corresponding JobExecuteResult as the parameter to the 'log' or 'status' methods of UJESClient. + +#### 3.Sample Implementation + +- **JAVA** +```java +package com.webank.bdp.dataworkcloud.ujes.client; + +import com.webank.wedatasphere.linkis.common.utils.Utils; +import com.webank.wedatasphere.linkis.httpclient.dws.authentication. +StaticAuthenticationStrategy; +import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfig; +import com.webank.wedatasphere.linkis.httpclient.dws.config.DWSClientConfigBuilder; +import com.webank.wedatasphere.linkis.ujes.client.UJESClient; +import com.webank.wedatasphere.linkis.ujes.client.UJESClientImpl; +import com.webank.wedatasphere.linkis.ujes.client.request.JobExecuteAction; +import com.webank.wedatasphere.linkis.ujes.client.request.ResultSetAction; +import com.webank.wedatasphere.linkis.ujes.client.response.JobExecuteResult; +import com.webank.wedatasphere.linkis.ujes.client.response.JobInfoResult; +import com.webank.wedatasphere.linkis.ujes.client.response.JobProgressResult; +import com.webank.wedatasphere.linkis.ujes.client.response.JobStatusResult; +import org.apache.commons.io.IOUtils; + +import java.util.concurrent.TimeUnit; + + +public class UJESClientImplTestJ{ + public static void main(String[] args){ + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder().addUJESServerUrl("http://${ip}:${port}") + .connectionTimeout(30000).discoveryEnabled(true) + .discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(true).maxConnectionSize(5) + .retryEnabled(false).readTimeout(30000) + .setAuthenticationStrategy(new StaticAuthenticationStrategy()).setAuthTokenKey("${username}") + .setAuthTokenValue("${password}"))).setDWSVersion("v1").build(); + UJESClient client = new UJESClientImpl(clientConfig); + + JobExecuteResult jobExecuteResult = client.execute(JobExecuteAction.builder().setCreator("UJESClient-Test") + .addExecuteCode("show tables") + .setEngineType(JobExecuteAction.EngineType$.MODULE$.HIVE()).setUser("${username}").build()); + System.out.println("execId: " + jobExecuteResult.getExecID() + ", taskId: " + jobExecuteResult.taskID()); + JobStatusResult status = client.status(jobExecuteResult); + while(!status.isCompleted()) { + JobProgressResult progress = client.progress(jobExecuteResult); + Utils.sleepQuietly(500); + status = client.status(jobExecuteResult); + } + JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); + String resultSet = jobInfo.getResultSetList(client)[0]; + Object fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser()).build()).getFileContent(); + System.out.println("fileContents: " + fileContents); + IOUtils.closeQuietly(client); + } +} +``` + +- **SCALA** +```scala + +import java.util.concurrent.TimeUnit + +import com.webank.wedatasphere.linkis.common.utils.Utils +import com.webank.wedatasphere.linkis.httpclient.dws.authentication. +StaticAuthenticationStrategy +import com.webank.wedatasphere.linkis.httpclient.dws.config. +DWSClientConfigBuilder +import com.webank.wedatasphere.linkis.ujes.client.request. +JobExecuteAction.EngineType +import com.webank.wedatasphere.linkis.ujes.client.request.{JobExecuteAction, ResultSetAction} +import org.apache.commons.io.IOUtils + +object UJESClientImplTest extends App { + + val clientConfig = DWSClientConfigBuilder.newBuilder().addUJESServerUrl("http://${ip}:${port}") + .connectionTimeout(30000).discoveryEnabled(true) + .discoveryFrequency(1, TimeUnit.MINUTES) + .loadbalancerEnabled(true).maxConnectionSize(5) + .retryEnabled(false).readTimeout(30000) + .setAuthenticationStrategy(new StaticAuthenticationStrategy()).setAuthTokenKey("${username}") + .setAuthTokenValue("${password}").setDWSVersion("v1").build() + val client = UJESClient(clientConfig) + + val jobExecuteResult = client.execute(JobExecuteAction.builder().setCreator("UJESClient-Test") + .addExecuteCode("show tables") + .setEngineType(EngineType.SPARK).setUser("${username}").build()) + println("execId: " + jobExecuteResult.getExecID + ", taskId: " + jobExecuteResult.taskID) + var status = client.status(jobExecuteResult) + while(!status.isCompleted) { + val progress = client.progress(jobExecuteResult) + val progressInfo = if(progress.getProgressInfo != null) progress.getProgressInfo.toList else List.empty + println("progress: " + progress.getProgress + ", progressInfo: " + progressInfo) + Utils.sleepQuietly(500) + status = client.status(jobExecuteResult) + } + val jobInfo = client.getJobInfo(jobExecuteResult) + val resultSet = jobInfo.getResultSetList(client).head + val fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser).build()).getFileContent + println("fileContents: " + fileContents) + IOUtils.closeQuietly(client) +} +``` diff --git a/docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md b/docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md new file mode 100644 index 00000000000..9c04153d87b --- /dev/null +++ b/docs/en_US/ch3/How_to_adapt_Linkis_with_a_new_computation_or_storage_engine.md @@ -0,0 +1,86 @@ +# How to adapt Linkis with a new computation or storage engine + +## 1. Introduction + +Except using the engines developed by Linkis directly, backend developers can also develop their own applications based on their requirements. +Divided into Entrance, EngineManager and Engine modules, one can easily split an application to adapt to Linkis. +The purpose and archetecture of these three modules please refer to Linkis Archetect Design Docs[UJES架构设计文档](../ch4/Linkis-UJES设计文档.md). + + +## 2. Declaration + +Linkis uses Spring framework as the underlying technique. So the Spring development specs must be obeyed. + +Linkis has an elastic underlying achitecture and provides common implementations for almost all of its top-level interfaces. If customized classes are needed by users, they can be directly injected and replace the current implementations. + + +### 2.1 Entrance module adaption + +**1) maven dependency** + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-entrance + 0.8.0 + +``` + +**2)Interfaces to be implemented** + +There is no compulsary interface in Entrance. Below interfaces can be implemented on demand. +- EntranceParser. Used to parse request maps from frontend to a persistable Task. Class AbstractEntranceParser is already provided and only parseToTask method needs to be overrided. Linkis provides CommonEntranceParser as the default implementation. +- EngineRequester. Used to build a RequestEngine object, which can be used to request a new engine from the EngineManager. +- Scheduler. Used to schedule tasks. The default implementation provides parallel mode for multi-user situations and FIFO mode for single user pattern. It is not suggested to be customized without special purposes. + +### 2.2 EngineManager module adaption + +**1) maven dependency** + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-enginemanager + 0.8.0 + +``` + +**2)Interfaces to be implemented** + +Below interfaces are required to be implemented in EngineManager: +- EngineCreator. Method createProcessEngineBuilder needs to be overridden in the existing AbstractEngineCreator to create an EngineBuilder. +Here ProcessEngineBuilder has already provided a class called JavaProcessEngineBuilder, which is an abstract class accomplishes configurations of classpath, JavaOpts, GC file path and log path, and opening DEBUG port in test mode. To implement JavaProcessEngineBuilder, only extra classpath and JavaOpts are needed to be specified. +- EngineResourceFactory. Method getRequestResource needs to be overridden in the existing AbstractEngineResourceFactory to declare user customized resource requirements. +- resources. A Spring bean used to register resources to RM. Users need to specify an instance of ModuleInfo for dependency injection. + +Below interfaces/beans are optional in EngineManager: +- hooks. A Spring bean used to add pre and post hooks around the Engine startup procedure. Users need to declare an Spring bean in type EngineHook[] hooks to make new hooks effective. For details please refer to com.webank.wedatasphere.linkis.enginemanager.impl.EngineManagerSpringConfiguration. + + +### 2.3 Engine module adaption + +**1) maven dependency** + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-engine + 0.8.0 + +``` + + +**2)Interfaces to be implemented** + +Below interfaces are required to be implemented in Engine： +- EngineExecutorFactory. Used to build an EngineExecutor from a Map by implementing method createExecutor. This map contains evironment variables and engine arguments. +- EngineExecutor. The actual executor to execute the code submitted from the entrance. + Methods need to be implemented: + 1. getActualUsedResources(the resource an engine acually used) + 2. executeLine(execute a line of the code parsed by CodeParser) + 3. executeCompletely(the suplementary method for executeLine. If executeLine returns ExecuteIncomplete, new code will be submitted with the previous code together to the engine) + +Below interfaces/beans are optional in Engine: +- engineHooks: Array[EngineHook], a Spring bean used to add pre and post hooks around the Engine startup procedure. Currently the system provides 2 hooks: CodeGeneratorEngineHook for UDF/Function loading and ReleaseEngineHook for releasing spare engines. The system registers engineHooks=Array(ReleaseEngineHook) only by default. +- CodeParser. Used to parse code into lines and submit one line only for each execution loop. The system registers a CodeParser returns all the code at once by default. +- EngineParser. Used to convert a RequestTask to a Job that is acceptable by Scheduler. If not specified, the system registers an EngineParser that converts RequestTask to CommonEngineJob. diff --git a/docs/en_US/ch3/Linkis User Manual.md b/docs/en_US/ch3/Linkis User Manual.md new file mode 100644 index 00000000000..637e33527d6 --- /dev/null +++ b/docs/en_US/ch3/Linkis User Manual.md @@ -0,0 +1,356 @@ +## Linkis User Manual + +#### 1.Summary +        Designed by Webank independently, Linkis is an extensible framework and a sophisticated solution for big data task submission. Conveniently, it could be used directly together with Scriptest, which is another open-source project powered by Webank. And those frontend APIs are also available for users. A client implementation is also provided as an SDK to interact directly with background services. As an highly extensible framework, users can leverage the SDK to develop their own applications. + +#### 2.Frontend Adaption +        Two protocols are supported for frontend APIs: HTTP and WebSocket. Compared with HTTP, Websocket is more friendly to servers and behaved more efficiently in message pushing. But WebSocket is unstable and users are easily to be disconnected by accident. So Scriptest combined both ways to adapt with Linkis. It communicates with Linkis by websocket in normal circumstances, and failover to HTTP protocol in case the Websocket connection was down. +##### 2.1 API Specs +Linkis has its own specs for front-backend adaption.
+ +**1).URL specs** +``` +/api/rest_j/v1/{applicationName}/.+ +/api/rest_s/v1/{applicationName}/.+ +``` + +- rest_j means the API is conformed to Jersey standards +- rest_s means the API is conformed to springMVC Rest standards +- v1 is the version of services，**The version will be upgraded with Linkis releases** +- {applicationName} is the microservice name + +**2).Request specs** +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{}, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + +**3).Response specs** +```json +{"method":"/api/rest_j/v1/entrance/execute","status":0, "message":Success！","data":{}} +``` +- method：Return the Restful API URL requested, basically used by websocket protocol。 +- status：Return the status info, in which -1 means login failed, 0 means succeeded, 1 means error, 2 mean validation failed, and 3 means no permission. +- data：Return detailed data. +- message：Return hint message of the request. If the status is not 0, this message returns error messages. At the same time 'data' may return the stack information in its 'stack' column. + + +##### 2.2WebSocket API Description + + +**1).Establish connection**
+
+Used to establish a WebSocket connection with Linkis. +- API `/api/rest_j/entrance/connect` +- HTTP Method **GET** +- Status Code **101**
+ +**2).Request execution**
+
+Used to submit user jobs to Linkis for execution. +- API `/api/rest_j/entrance/execute` +- HTTP Method `POST`
+- Sample Json request body +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{ + "params": { + "variable":{ + "k1":"v1" + }, + "configuration":{ + "special":{ + "k2":"v2" + }, + "runtime":{ + "k3":"v3" + }, + "startup":{ + "k4":"v4" + } + } + }, + "executeApplicationName":"spark", + "executionCode":"show tables", + "runType":"sql", + "source":{ + "scriptPath": "/home//linkis.sql" + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" + } +} +``` +- Descriptions for parameters of request body data
+ +| Parameter Name | Parameter Definition | Type | Comments | +| ------------ | ------------ | ------------ | ------------ | +| executeApplicationName | The Engine service expected by the user, such as Spark or hive| String | Not null | +| requestApplicationName | The name of the system launching this request | String | Nullable | +| params | User-defined parameters to run services | Map | Required, but values are nullable | +| executionCode | The execution code submitted by the user | String |Not null | +| runType | Assuming that the user executes a spark job, he may choose python, R or SQL as runType| String | Not null | +| scriptPath | The script path of the execution code | String | For Scriptest, it shouldn't be null with executionCode at the same time | + Table 1 Descriptions for the parameters + + +- Sample Json response body +```json +{ + "method": "/api/rest_j/v1/entrance/execute", + "status": 0, + "message": "Execution request succeeded", + "data": { + "execID": "030418IDEhivebdpdwc010004:10087IDE_johnnwang_21", + "taskID": "123" + } +} +``` +- execID is a unique ID of String type generated for each user task after submitted to Linkis. It is only used during the execution period, like PID. The format of execID is (length of requestApplicationName)(length of executeAppName)(length of Instance)${requestApplicationName}${executeApplicationName}${entranceInstance infomation ip+port}${requestApplicationName}_${umUser}_${index} +- taskID is a unique ID of Long type genenrated incrementally by the database for each task. + +**3).The push mechanism for task status, logs and progress**
+ +After submission, the status, logs and progress information will be pushed by the server. They could be retrieved by websocket protocol. +The API is consistent with HTTP protocol as mentioned below. The only difference is that the schema of websocket is ws://, but http:// for HTTP protocol. + +Sample response of WebSocket API +- Logs +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "Returned log information", + "data": { + "execID": "${execID}", + "log": ["errorLog","warnLog","infoLog", "allLog"], + "taskID":28594, + "fromLine": 56 + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` +- Status +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/status", + "status": 0, + "message": "Return status information", + "data": { + "execID": "${execID}", + "taskID":28594, + "status": "Running", + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` +- Progress +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "Return progress information", + "data": { + "execID": "${execID}", + "taskID":28594, + "progress": 0.2, + "progressInfo": [ + { + "id": "job-1", + "succeedTasks": 2, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + }, + { + "id": "job-2", + "succeedTasks": 5, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + } + ] + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + + + +##### 2.3HTTP API description +For HTTP API, polling should be used to retrieve the status, logs and progress information after submission. + + +**1).Request execution** + +- API `/api/rest_j/entrance/execute` +- HTTP Method `POST`
+- Sample JSON request body, its description is the same with Table 1 +```json +{ + "params": {}, + "executeApplicationName":"spark", + "executionCode":"show tables", + "runType":"sql", + "source":{ + "scriptPath": "/home/linkis/linkis.sql" + } +} +``` +- The response is consistent with websocket. Both execId and taskId will be obtained. + +**2).Retrieve status**
+
+- API `/api/rest_j/entrance/${execID}/status` +- HTTP Method `GET`
+- Sample response body +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/status", + "status": 0, + "message": "Succeeded to retrieve status", + "data": { + "execID": "${execID}", + "status": "Running" + } +} +``` + +**3).Retrieve logs**
+
+- API `/api/rest_j/entrance/${execID}/log?fromLine=${fromLine}&size=${size}` +- HTTP Method `GET` +- Parameter fromLine specifies from which line to start. Parameter size specifies the number of lines should be retrieved for this request. +- Sample response body, the returned fromLine indicates the value of parameter fromLine for next request. +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "Return logs information", + "data": { + "execID": "${execID}", + "log": ["errorLogs","warnLogs","infoLogs", "allLogs], + "fromLine": 56 + } +} +``` + +**4).Retrieve progress**
+
+- API `/api/rest_j/entrance/${execID}/progress` +- HTTP Method `GET`
+- Sample response body +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/progress", + "status": 0, + "message": "Return progress information", + "data": { + "execID": "${execID}", + "progress": 0.2, + "progressInfo": [ + { + "id": "job-1", + "succeedTasks": 2, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + }, + { + "id": "job-2", + "succeedTasks": 5, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + } + ] + } +} +``` +**5).kill task**
+
+- API `/api/rest_j/entrance/${execID}/kill` +- HTTP Method `POST` +- Sample response body +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/kill", + "status": 0, + "message": "OK", + "data": { + "execID":"${execID}" + } +} +``` + + + +### 3.Client SDK Adaption +[Please see Linkis Quick Start](/docs/en_US/ch2/Linkis%20Quick%20Start.md) + + + +#### 4. Multiple engine type support +Except using the engines developed by Linkis directly, backend developers can also develop their own applications based on their requirements. Divided into Entrance, EngineManager and Engine modules, one can easily split an application to adapt to Linkis. The purpose and architecture of these three modules please refer to Linkis Architect Design Docs. + +#### Convention + +Linkis uses Spring framework as the underlying technique. So instances of some classes can be injected using Spring annotations. Linkis provides some default common implementations. If customized classes are needed by users, they can be directly injected and replace the current implementations. + +##### 4.1Entrance module adaption +**1)maven dependency** +```xml + + com.webank.wedatasphere.linkis + linkis-ujes-entrance + 0.8.0 + +``` +**2)Interfaces to be implemented** + +There is no compulsory interface in Entrance. Below interfaces can be implemented on demand. +- EntranceParser. Used to parse request maps from frontend to a persistable Task. Class AbstractEntranceParser is already provided and only parseToTask method needs to be overridden. Linkis provides CommonEntranceParser as the default implementation. +- EngineRequester. Used to build a RequestEngine object, which can be used to request a new engine from the EngineManager. +- Scheduler. Used to schedule tasks. The default implementation provides parallel mode for multi-user situations and FIFO mode for single user pattern. It is not suggested to be customized without special purposes. + +##### 4.2EngineManager module adaption +**1)maven dependency** +```xml + + com.webank.wedatasphere.linkis + linkis-ujes-enginemanager + 0.8.0 + +``` + +**2)Interfaces to be implemented** + +Below interfaces are required to be implemented in EngineManager: +- EngineCreator. Method createProcessEngineBuilder needs to be overridden in the existing AbstractEngineCreator to create an EngineBuilder. +Here ProcessEngineBuilder has already provided a class called JavaProcessEngineBuilder, which is an abstract class accomplishes configurations of classpath, JavaOpts, GC file path and log path, and opening DEBUG port in test mode. To implement JavaProcessEngineBuilder, only extra classpath and JavaOpts are needed to be specified. +- EngineResourceFactory. Method getRequestResource needs to be overridden in the existing AbstractEngineResourceFactory to declare user customized resource requirements. +- hooks. A Spring bean used to add pre and post hooks around the Engine startup procedure. Users need to specify an Array[EngineHook] for dependency injection. +- resources. A Spring bean used to register resources to RM. Users need to specify an instance of ModuleInfo for dependency injection. + +##### 4.3Engine module adaption +**1)maven dependency** +```xml + + com.webank.wedatasphere.linkis + linkis-ujes-engine + 0.8.0 + +``` +**2)Interfaces to be implemented** + +Below interfaces are required to be implemented in Engine： +- EngineExecutorFactory. Used to build an EngineExecutor from a Map by implementing method createExecutor. +- EngineExecutor. The actual executor to execute the code submitted from the entrance. Methods need to be implemented: getActualUsedResources(the resource an engine acually used), executeLine(execute a line of the code parsed by CodeParser), executeCompletely(the suplementary method for executeLine. If executeLine returns ExecuteIncomplete, new code will be submitted with the previous code together to the engine) + +Below interfaces/beans are optional in Engine: +- engineHooks: Array[EngineHook], a Spring bean used to add pre and post hooks around the Engine startup procedure. Currently the system provides 2 hooks: CodeGeneratorEngineHook for UDF/Function loading and ReleaseEngineHook for releasing spare engines. The system registers engineHooks=Array(ReleaseEngineHook) only by default. +- CodeParser. Used to parse code into lines and submit one line only for each execution loop. The system registers a CodeParser returns all the code at once by default. +- EngineParser. Used to convert a RequestTask to a Job that is acceptable by Scheduler. If not specified, the system registers an EngineParser that converts RequestTask to CommonEngineJob. \ No newline at end of file diff --git a/docs/en_US/ch3/Linkis_HTTP_API_Doc.md b/docs/en_US/ch3/Linkis_HTTP_API_Doc.md new file mode 100644 index 00000000000..8befe455e1a --- /dev/null +++ b/docs/en_US/ch3/Linkis_HTTP_API_Doc.md @@ -0,0 +1,193 @@ +## Linkis HTTP API Doc + + +#### 1.Summary +        Linkis provides an adaption method by HTTP, for the convenience of the frontend of functional applications. + +The data development IDE tool [Scriptis](https://github.com/WeBankFinTech/Scriptis) combined both ways to adapt with Linkis. It communicates with Linkis by websocket in normal circumstances, and failover to HTTP protocol in case the Websocket connection was down. + +##### 2.1 API Specs +Linkis defined its own specs for front-backend adaption.
+ +**1).URL specs** +``` +/api/rest_j/v1/{applicationName}/.+ +/api/rest_s/v1/{applicationName}/.+ +``` + +- rest_j means the API is conformed to Jersey standards +- rest_s means the API is conformed to springMVC Rest standards +- v1 is the version of services，**The version will be upgraded with Linkis releases** +- {applicationName} is the microservice name + +**2).Request specs** +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{}, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + +**3).Response specs** +```json +{"method":"/api/rest_j/v1/entrance/execute","status":0, "message":Success！","data":{}} +``` +- method：Return the Restful API URL requested, basically used by websocket protocol。 +- status：Return the status info, in which -1 means login failed, 0 means succeeded, 1 means error, 2 mean validation failed, and 3 means no permission. +- data：Return detailed data. +- message：Return hint message of the request. If the status is not 0, this message returns error messages. At the same time 'data' may return the stack information in its 'stack' column. + + +##### 2.2 HTTP API description +For HTTP API, polling should be used to retrieve the status, logs and progress information after submission. + + +**1).Request execution** + +- API `/api/rest_j/entrance/execute` +- HTTP Method `POST` + +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{ + "params": { + "variable":{ + "k1":"v1" + }, + "configuration":{ + "special":{ + "k2":"v2" + }, + "runtime":{ + "k3":"v3" + }, + "startup":{ + "k4":"v4" + } + } + }, + "executeApplicationName":"spark", + "executionCode":"show tables", + "runType":"sql", + "source":{ + "scriptPath": "/home/Linkis/Linkis.sql" + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" + } +} +``` +-Description of parameters in the request body data + +| Parameter Name | Parameter Definition | Type | Comments | +| ------------ | ------------ | ------------ | ------------ | +| executeApplicationName | The Engine service expected by the user, such as Spark or hive| String | Not null | +| requestApplicationName | The name of the system launching this request | String | Nullable | +| params | User-defined parameters to run services | Map | Required, but values are nullable | +| executionCode | The execution code submitted by the user | String |Not null | +| runType | Assuming that the user executes a spark job, he may choose python, R or SQL as runType| String | Not null | +| scriptPath | The script path of the execution code | String | For Scriptest, it shouldn't be null with executionCode at the same time | + Table 1 Descriptions for the parameters + +- Sample Json response body +```json +{ + "method": "/api/rest_j/v1/entrance/execute", + "status": 0, + "message": "Execution request succeeded", + "data": { + "execID": "030418IDEhivebdpdwc010004:10087IDE_johnnwang_21", + "taskID": "123" + } +} +``` +- execID is a unique ID of String type generated for each user task after submitted to Linkis. It is only used during the execution period, like PID. The format of execID is (length of requestApplicationName)(length of executeAppName)(length of Instance)${requestApplicationName}${executeApplicationName}${entranceInstance infomation ip+port}${requestApplicationName}_${umUser}_${index} +- taskID is a unique ID of Long type genenrated incrementally by the database for each task. + + + + +**2).Retrieve status**
+
+- API `/api/rest_j/entrance/${execID}/status` +- HTTP Method `GET`
+- Sample response body +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/status", + "status": 0, + "message": "Succeeded to retrieve status", + "data": { + "execID": "${execID}", + "status": "Running" + } +} +``` + +**3).Retrieve logs**
+
+- API `/api/rest_j/entrance/${execID}/log?fromLine=${fromLine}&size=${size}` +- HTTP Method `GET` +- Parameter fromLine specifies from which line to start. Parameter size specifies the number of lines should be retrieved for this request. +- Sample response body, the returned fromLine indicates the value of parameter fromLine for next request. +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "Return logs information", + "data": { + "execID": "${execID}", + "log": ["errorLogs","warnLogs","infoLogs", "allLogs], + "fromLine": 56 + } +} +``` + +**4).Retrieve progress**
+
+- API `/api/rest_j/entrance/${execID}/progress` +- HTTP Method `GET`
+- Sample response body +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/progress", + "status": 0, + "message": "Return progress information", + "data": { + "execID": "${execID}", + "progress": 0.2, + "progressInfo": [ + { + "id": "job-1", + "succeedTasks": 2, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + }, + { + "id": "job-2", + "succeedTasks": 5, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + } + ] + } +} +``` +**5).kill task**
+
+- API `/api/rest_j/entrance/${execID}/kill` +- HTTP Method `POST` +- Sample response body +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/kill", + "status": 0, + "message": "OK", + "data": { + "execID":"${execID}" + } +} +``` diff --git a/docs/en_US/ch3/Linkis_Introduction.md b/docs/en_US/ch3/Linkis_Introduction.md new file mode 100644 index 00000000000..6fa2d2642c1 --- /dev/null +++ b/docs/en_US/ch3/Linkis_Introduction.md @@ -0,0 +1,62 @@ +# Linkis Introduction + +## 1. Summary + +Linkis is the data middleware as a part of [WeDataSphere](https://github.com/WeBankFinTech/WeDataSphere), the big data suitcase of WeBank. + +---- + +## 2. Background + +With the widely spreading usage of big data technology, the open-source community has breeded out a variety of concrete applications and underlying computation engines. + +Currently, it is the common solution for all the enterprises to introduce multiple open-source components for different business requirements, keeping their big data platform architecture refreshed and affluent. + +As shown in the below diagram, with the increased number of concrete applications, tooling systems and underlying computation/storage engines, the whole data platform turns to a reticulation structure. +
+ +![Original big data ecosystem diagram](../../zh_CN/images/ch3/原大数据生态图.jpg) + +
+At the same time of introducing new open-source components to solve the pain points, those new components must be adapted with different functional applications or underlying computation/storage engines, leading to more pain points: + +1. The bussiness requirements are keep changing, but every open-source component is distinguished in its features. Therefore, users may feel fragile and expensive both to learn and to use. +2. The diversity in big data types increases the difficulty of computation and storage. To keep up with those emerging computation/storage engines, developers must have sufficient technology stack. +3. Introducing new open-source components will inevitably cause compatibility issues with the existing data platform in multi-tenant isolation, user resource management and development management. And it is a heavy work to do top-to-bottom customized development, with problems of "re-inventing the wheel" and long development cycles. +4. Most data platform connects the functional applications with the underlying engines so directly that once exceptions occur, those unpredictable strange problems have negative effects on the user experience. It is also hard to gain any sediment on operation & maintenance. + + +---- + +## 3. Our Exploration + +To construct a enterprise-level big data platform with sufficient & powerful functional tools and application systems, below questions should be taken into concern: +
+ +![Questions to concern](../../zh_CN/images/ch3/要解决的问题.jpg) +
+ +1. How to immediately gain the enterprise-level capabilities of multi-tenant isolation, elastic scaling and resource governance? +2. How to reuse those platform level capabilities and achieve the fast adaption with newly introduced components? +3. How to simplify the adaption with functional applications in a convenient and standardized style? No more tolerance with a bunch of Maven dependencies about Hadoop, Hive and Spark together with solving environment and argument issues. +4. How to solve the problem of application islet and data islet? How to quickly publish and schedule the execution of the code developed by IDE tools? How to make data consanguinity service one-stop fully covered? + +We redefined the "connection" layer of data platforms and provided our exclusive solution based on above pain points. + + +---- + +### 4. Solution + +Linkis connects with computation/storage engines(Spark, Flink, Hive, Python and HBase), exposes REST/WebSocket/JDBC interface, and executes multi-language jobs(SQL, Pyspark, HiveQL and Scala), as a data middleware. + + +![Solution](../../zh_CN/images/ch3/解决方案.jpg) + +
+Linkis provides below common modules to abstract, analyze and solve most problems on big data platform layer: +
+ +![Functions and modules diagram](../../zh_CN/images/ch3/功能模块图.png) + +Linkis, the edge-cutting tool to build up a finance and enterprise level big data platforms! diff --git a/docs/en_US/ch3/Linkis_Java_SDK_doc.md b/docs/en_US/ch3/Linkis_Java_SDK_doc.md new file mode 100644 index 00000000000..2dc782d5931 --- /dev/null +++ b/docs/en_US/ch3/Linkis_Java_SDK_doc.md @@ -0,0 +1,157 @@ +### Linkis Java SDK Doc +#### 1.Introduction + +Linkis provides a Java client implementation for users to have quick-access to Linkis background services using UJESClient. + +#### 2 Quick start + + We provided two test classes under dir ujes/client/src/test： + + com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTestJ # Test class based on Java + com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTest # Test class based on Scala + + If you have cloned the source code of Linkis, you can directly run these two test classes. + + + **Below sections introduce about how to write the code to complete a single execution on Linkis** + +#### 3 Fast implementation + +##### 3.1 maven dependency + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-client + 0.8.0 + +``` + +#### 3.2 Sample implementation + +- **JAVA** +```java +package com.webank.bdp.dataworkcloud.ujes.client; + +import com.webank.wedatasphere.Linkis.common.utils.Utils; +import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy; +import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfig; +import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder; +import com.webank.wedatasphere.Linkis.ujes.client.UJESClient; +import com.webank.wedatasphere.Linkis.ujes.client.UJESClientImpl; +import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction; +import com.webank.wedatasphere.Linkis.ujes.client.request.ResultSetAction; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobExecuteResult; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobInfoResult; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobProgressResult; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobStatusResult; +import org.apache.commons.io.IOUtils; + +import java.util.concurrent.TimeUnit; + + +public class UJESClientImplTestJ{ + public static void main(String[] args){ + // 1. To do the configuration, an instance of DWSClientConfig should be obtained from DWSClientBuilder. + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder() + .addUJESServerUrl("http://${ip}:${port}") //Specify the ServerUrl，The address of Willink gateway, i.e. http://{ip}:{port} + .connectionTimeout(30000) //connectionTimeOut: The connection timeout of the client + .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //Enable service discovery. Once enabled, newly started Gateway will be auto-dicovered. + .loadbalancerEnabled(true) // Enable load balancing. Cannot be enabled alone without service discovery enabled. + .maxConnectionSize(5) //Max connection size, aka the max concurrent threshold + .retryEnabled(false).readTimeout(30000) //whether to retry after failure + .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy, The authentication strategy of Linkis + .setAuthTokenKey("${username}").setAuthTokenValue("${password}"))) //The authentication key，usually the username;The authentication value，usually the password + .setDWSVersion("v1").build(); //The version of Linkis background protocol, currently v1 + + // 2. Create a UJESClient from DWSClientConfig + UJESClient client = new UJESClientImpl(clientConfig); + + // 3. Begin to execute the code + JobExecuteResult jobExecuteResult = client.execute(JobExecuteAction.builder() + .setCreator("LinkisClient-Test") //creator. The name of the system which holds the UJES client, used for system level isolation. + .addExecuteCode("show tables") //ExecutionCode. The code which is requested to be executed + .setEngineType(JobExecuteAction.EngineType$.MODULE$.HIVE()) // The engine type expected by the client, i.e. Spark, Hive, etc... + .setUser("johnnwang") //User, The user who makes this request；Used for user level multi-tenant isolation + .build()); + System.out.println("execId: " + jobExecuteResult.getExecID() + ", taskId: " + jobExecuteResult.taskID()); + + // 4. Synch the status of script execution + JobStatusResult status = client.status(jobExecuteResult); + while(!status.isCompleted()) { + // 5. Synch the status of script execution + JobProgressResult progress = client.progress(jobExecuteResult); + Utils.sleepQuietly(500); + status = client.status(jobExecuteResult); + } + + // 6. Synch the job information of script execution + JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); + // 7. Fetch the list of result sets(Multiple result sets will be generated if a user submitted multiple SQL at once) + String resultSet = jobInfo.getResultSetList(client)[0]; + // 8. Fetch detailed result set content with a particular result set info + Object fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser()).build()).getFileContent(); + System.out.println("fileContents: " + fileContents); + IOUtils.closeQuietly(client); + } +} +``` + +- **SCALA** +```scala + +import java.util.concurrent.TimeUnit + +import com.webank.wedatasphere.Linkis.common.utils.Utils +import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy +import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder +import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction.EngineType +import com.webank.wedatasphere.Linkis.ujes.client.request.{JobExecuteAction, ResultSetAction} +import org.apache.commons.io.IOUtils + +object UJESClientImplTest extends App { + + // 1. To do the configuration, an instance of DWSClientConfig should be obtained from DWSClientBuilder. + val clientConfig = DWSClientConfigBuilder.newBuilder() + .addUJESServerUrl("http://${ip}:${port}") //Specify the ServerUrl，The address of Willink gateway, i.e. http://{ip}:{port} + .connectionTimeout(30000) //connectionTimeOut: The connection timeout of the client + .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //Enable service discovery. Once enabled, newly started Gateway will be auto-dicovered. + .loadbalancerEnabled(true) // Enable load balancing. Cannot be enabled alone without service discovery enabled. + .maxConnectionSize(5) //Max connection size, aka the max concurrent threshold + .retryEnabled(false).readTimeout(30000) //whether to retry after failure + .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy, The authentication strategy of Linkis + .setAuthTokenKey("${username}").setAuthTokenValue("${password}") //The authentication key，usually the username;The authentication value，usually the password + .setDWSVersion("v1").build() //The version of Linkis background protocol, currently v1 + + // 2. Create a UJESClient from DWSClientConfig + val client = UJESClient(clientConfig) + + // 3. Begin to execute the code + val jobExecuteResult = client.execute(JobExecuteAction.builder() + .setCreator("LinkisClient-Test") //creator. The name of the system which holds the UJES client, used for system level isolation. + .addExecuteCode("show tables") //ExecutionCode. The code which is requested to be executed + .setEngineType(EngineType.SPARK) // The engine type expected by the client, i.e. Spark, Hive, etc... + .setUser("${username}").build()) //User, The user who makes this request；Used for user level multi-tenant isolation + println("execId: " + jobExecuteResult.getExecID + ", taskId: " + jobExecuteResult.taskID) + + // 4. Synch the status of script execution + var status = client.status(jobExecuteResult) + while(!status.isCompleted) { + // 5. Synch the status of script execution + val progress = client.progress(jobExecuteResult) + val progressInfo = if(progress.getProgressInfo != null) progress.getProgressInfo.toList else List.empty + println("progress: " + progress.getProgress + ", progressInfo: " + progressInfo) + Utils.sleepQuietly(500) + status = client.status(jobExecuteResult) + } + + // 6. Synch the job information of script execution + val jobInfo = client.getJobInfo(jobExecuteResult) + // 7. Fetch the list of result sets(Multiple result sets will be generated if a user submitted multiple SQL at once) + val resultSet = jobInfo.getResultSetList(client).head + // 8. Fetch detailed result set content with a particular result set info + val fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser).build()).getFileContent + println("fileContents: " + fileContents) + IOUtils.closeQuietly(client) +} +``` diff --git a/docs/en_US/ch3/Linkis_WebSocket_API_Doc.md b/docs/en_US/ch3/Linkis_WebSocket_API_Doc.md new file mode 100644 index 00000000000..a69852bb4d9 --- /dev/null +++ b/docs/en_US/ch3/Linkis_WebSocket_API_Doc.md @@ -0,0 +1,180 @@ +## Linkis WebSocket API Doc + + +#### 1.Summary +        Linkis provides an adaption method by WebSocket, for the convenience of the frontend of functional applications. + +The data development IDE tool [Scriptis](https://github.com/WeBankFinTech/Scriptis) combined both ways to adapt with Linkis. It communicates with Linkis by websocket in normal circumstances, and failover to HTTP protocol in case the Websocket connection was down. + +##### 2.1 API Specs +Linkis defined its own specs for front-backend adaption.
+ +**1).URL specs** +``` +/api/rest_j/v1/{applicationName}/.+ +/api/rest_s/v1/{applicationName}/.+ +``` + +- rest_j means the API is conformed to Jersey standards +- rest_s means the API is conformed to springMVC Rest standards +- v1 is the version of services，**The version will be upgraded with Linkis releases** +- {applicationName} is the microservice name + +**2).Request specs** +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{}, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + +**3).Response specs** +```json +{"method":"/api/rest_j/v1/entrance/execute","status":0, "message":Success！","data":{}} +``` +- method：Return the Restful API URL requested, basically used by websocket protocol。 +- status：Return the status info, in which -1 means login failed, 0 means succeeded, 1 means error, 2 mean validation failed, and 3 means no permission. +- data：Return detailed data. +- message：Return hint message of the request. If the status is not 0, this message returns error messages. At the same time 'data' may return the stack information in its 'stack' column. + + +##### 2.2WebSocket API Description + + +**1).Establish connection**
+
+Used to establish a WebSocket connection with Linkis. +- API `/api/rest_j/entrance/connect` +- HTTP Method **GET** +- Status Code **101**
+ +**2).Request execution**
+
+Used to submit user jobs to Linkis for execution. +- API `/api/rest_j/entrance/execute` +- HTTP Method `POST`
+- Sample Json request body +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{ + "params": { + "variable":{ + "k1":"v1" + }, + "configuration":{ + "special":{ + "k2":"v2" + }, + "runtime":{ + "k3":"v3" + }, + "startup":{ + "k4":"v4" + } + } + }, + "executeApplicationName":"spark", + "executionCode":"show tables", + "runType":"sql", + "source":{ + "scriptPath": "/home/Linkis/Linkis.sql" + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" + } +} +``` +- Descriptions for parameters of request body data
+ +| Parameter Name | Parameter Definition | Type | Comments | +| ------------ | ------------ | ------------ | ------------ | +| executeApplicationName | The Engine service expected by the user, such as Spark or hive| String | Not null | +| requestApplicationName | The name of the system launching this request | String | Nullable | +| params | User-defined parameters to run services | Map | Required, but values are nullable | +| executionCode | The execution code submitted by the user | String |Not null | +| runType | Assuming that the user executes a spark job, he may choose python, R or SQL as runType| String | Not null | +| scriptPath | The script path of the execution code | String | For Scriptest, it shouldn't be null with executionCode at the same time | + Table 1 Descriptions for the parameters + + +- Sample Json response body +```json +{ + "method": "/api/rest_j/v1/entrance/execute", + "status": 0, + "message": "Execution request succeeded", + "data": { + "execID": "030418IDEhivebdpdwc010004:10087IDE_johnnwang_21", + "taskID": "123" + } +} +``` +- execID is a unique ID of String type generated for each user task after submitted to Linkis. It is only used during the execution period, like PID. The format of execID is (length of requestApplicationName)(length of executeAppName)(length of Instance)${requestApplicationName}${executeApplicationName}${entranceInstance infomation ip+port}${requestApplicationName}_${umUser}_${index} +- taskID is a unique ID of Long type genenrated incrementally by the database for each task. + +**3).The push mechanism for task status, logs and progress**
+ +After submission, the status, logs and progress information will be pushed by the server. They could be retrieved by websocket protocol. +The API is consistent with HTTP protocol as mentioned below. The only difference is that the schema of websocket is ws://, but http:// for HTTP protocol. + +Sample response of WebSocket API +- Logs +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "Returned log information", + "data": { + "execID": "${execID}", + "log": ["errorLog","warnLog","infoLog", "allLog"], + "taskID":28594, + "fromLine": 56 + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` +- Status +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/status", + "status": 0, + "message": "Return status information", + "data": { + "execID": "${execID}", + "taskID":28594, + "status": "Running", + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` +- Progress +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "Return progress information", + "data": { + "execID": "${execID}", + "taskID":28594, + "progress": 0.2, + "progressInfo": [ + { + "id": "job-1", + "succeedTasks": 2, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + }, + { + "id": "job-2", + "succeedTasks": 5, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + } + ] + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` diff --git a/docs/en_US/images/introduction/introduction01.png b/docs/en_US/images/introduction/introduction01.png new file mode 100644 index 00000000000..f1f68410413 Binary files /dev/null and b/docs/en_US/images/introduction/introduction01.png differ diff --git a/docs/en_US/images/introduction/introduction02.png b/docs/en_US/images/introduction/introduction02.png new file mode 100644 index 00000000000..5bb96cdde3e Binary files /dev/null and b/docs/en_US/images/introduction/introduction02.png differ diff --git a/docs/en_US/images/introduction/introduction03.jpg b/docs/en_US/images/introduction/introduction03.jpg new file mode 100644 index 00000000000..b74ab42601f Binary files /dev/null and b/docs/en_US/images/introduction/introduction03.jpg differ diff --git a/docs/en_US/images/introduction/introduction04.jpg b/docs/en_US/images/introduction/introduction04.jpg new file mode 100644 index 00000000000..0a05bd2e351 Binary files /dev/null and b/docs/en_US/images/introduction/introduction04.jpg differ diff --git a/docs/en_US/images/introduction/introduction05.png b/docs/en_US/images/introduction/introduction05.png new file mode 100644 index 00000000000..01ec041d7f1 Binary files /dev/null and b/docs/en_US/images/introduction/introduction05.png differ diff --git a/docs/zh_CN/README.md b/docs/zh_CN/README.md new file mode 100644 index 00000000000..db5792b900f --- /dev/null +++ b/docs/zh_CN/README.md @@ -0,0 +1,128 @@ +Linkis +============ + +[![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) + +[English](../../README.md) | 中文 + +# 引言： + +Linkis是一个打通了多个计算存储引擎如：Spark、TiSpark、Hive、Python和HBase等，对外提供统一REST/WebSocket/JDBC接口，提交执行SQL、Pyspark、HiveQL、Scala等脚本的数据中间件。 + +Linkis基于微服务架构，提供了金融级多租户隔离、资源管控、权限隔离等企业级特性，支持统一变量、UDF、函数、用户资源文件管理，具备高并发、高性能、高可用的大数据作业/请求全生命周期管理能力。 + + +![Linkis效果图](images/ch3/解决方案.jpg) + +---- + +# 核心特点： + +- **统一作业执行服务**：一个分布式的REST/WebSocket服务，用于接收用户提交的各种脚本请求。 + + **目前支持的计算引擎有**：Spark、Python、TiSpark、Hive和Shell等。 + + **支持的脚本语言有**：SparkSQL、Spark Scala、Pyspark、R、Python、HQL和Shell等； + + + +- **资源管理服务**： 支持实时管控每个系统和用户的资源使用情况，限制系统和用户的资源使用量和并发数，并提供实时的资源动态图表，方便查看和管理系统和用户的资源； + + **目前已支持的资源类型**：Yarn队列资源、服务器（CPU和内存）、用户并发个数等。 + + + +- **应用管理服务**：管理所有系统的所有用户应用，包括离线批量应用、交互式查询应用和实时流式应用，为离线和交互式应用提供强大的复用能力，并提供应用全生命周期管理，自动释放用户多余的空闲应用； + + + +- **统一存储服务**：通用的IO架构，能快速对接各种存储系统，提供统一调用入口，支持所有常用格式数据，集成度高，简单易用； + + + +- **统一上下文服务**：统一用户和系统资源文件（JAR、ZIP、Properties等），用户、系统、计算引擎的参数和变量统一管理，一处设置，处处自动引用； + + + +- **物料库**：系统和用户级物料管理，可分享和流转，支持全生命周期自动管理； + + + +- **元数据服务**：实时的库表结构和分区情况展示。 + +## 与类似系统的对比： + +Linkis跟Apache Livy和Apache Zeppelin Interpreter的定位不一样，但是由于开源社区目前尚没有同类竞品，所以这里做一个简单对比供大家参考。 + +![comparison_table](images/introduction/comparison_table.png) + + + +---- + +# 文档列表 + +[Linkis快速安装使用文档](https://github.com/WeBankFinTech/Linkis/wiki/%E5%A6%82%E4%BD%95%E5%BF%AB%E9%80%9F%E5%AE%89%E8%A3%85%E4%BD%BF%E7%94%A8Linkis) + +[上层前端应用的HTTP接入文档](https://github.com/WeBankFinTech/Linkis/wiki/%E4%B8%8A%E5%B1%82%E5%89%8D%E7%AB%AF%E5%BA%94%E7%94%A8HTTP%E6%8E%A5%E5%85%A5%E6%96%87%E6%A1%A3) + +[上层前端应用的WebSocket接入文档](https://github.com/WeBankFinTech/Linkis/wiki/%E4%B8%8A%E5%B1%82%E5%89%8D%E7%AB%AF%E5%BA%94%E7%94%A8WebSocket%E6%8E%A5%E5%85%A5%E6%96%87%E6%A1%A3) + +[Linkis架构文档](https://github.com/WeBankFinTech/Linkis/wiki/%E6%80%BB%E4%BD%93%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1%E4%BB%8B%E7%BB%8D) + +**更多文档，请[点我访问](https://github.com/WeBankFinTech/Linkis/wiki)** + + +---- + +# Architecture： + +![introduction_new](images/introduction/introduction_new.png) + +---- + +# RoadMap + +### 1. Linkis管理台 + +- 用户可以查看和管理自己的Job +- 用户可以查看和管理自己的引擎 +- 用户可以查看Yarn的资源使用情况和用户资源使用排名 +- 用户可以配置引擎的启动参数 +- 用户可以设置变量 + +### 2. 对外接口，提供JDBC通用接口 + +丰富Linkis的对外接口，外部用户可以通过JDBC的方式，访问Linkis。 + +### 3. 底层的计算存储引擎，新增对JDBC的支持 + +实现一个全新的JDBC引擎，底层支持直接对接MySQL、TiDB等。 + +### 4. 适配Spark2.2以上的版本 + +适配Spark2.2以上的版本，支持Spark现有的所有版本 + +
+ +**如果您有任何需求，欢迎给我们提issue，我们将会及时给您回复。** + +---- + +# Contributing + +非常欢迎广大的社区伙伴给我们贡献新引擎和代码！ + +---- + +# Communication + +如果您想得到最快的响应，请给我们提issue，或者您也可以扫码进群： + +![introduction05](images/introduction/introduction05.png) + +---- + +# License + +**Linkis is under the Apache 2.0 license. See the [LICENSE](/LICENSE) file for details.** diff --git a/docs/zh_CN/SUMMARY.md b/docs/zh_CN/SUMMARY.md new file mode 100644 index 00000000000..391a4e1e366 --- /dev/null +++ b/docs/zh_CN/SUMMARY.md @@ -0,0 +1,16 @@ +# Summary + +* [Introduction](README.md) +* [系统部署文档](ch1/deploy.md) +* [快速使用文档](ch2/linkis快速使用文档.md) +* [系统使用手册](ch3/linkis使用文档.md) +* [架构设计]() + * [UJES架构设计](ch4/Linkis-UJES设计文档.md) + * [RM架构设计](ch4/Linkis-RM设计文档.md) + * [AM架构设计]() +* [开发文档]() +* [FAQ]() +* [版本升级文档]() +* [历次版本发布内容]() + + diff --git a/docs/zh_CN/ch1/deploy.md b/docs/zh_CN/ch1/deploy.md new file mode 100644 index 00000000000..894d3a32462 --- /dev/null +++ b/docs/zh_CN/ch1/deploy.md @@ -0,0 +1,136 @@ +## 1、准备工作 + +### (1) 基础软件安装(必装项请自行安装) + +- Mysql (5.5+) : 必装 +- JDK (1.8.0_141) : 必装 +- Hadoop(**社区版3.0以下都支持，CDH版可能存在兼容性问题**) ：选装，如需使用hive\spark引擎功能则需要安装Hadoop，只使用Python无需安装 +- Hive(1.2.1，**内部没有测过2.0和2.0以上版本，可能存在兼容性问题**) : 选装，hive引擎节点需要安装 +- Spark(**只支持2.2.0以下，暂不支持Spark2.2以上的版本**) : 选装，Spark引擎节点需要安装 +- Python(2.x和3.x都支持)：选装，使用python引擎时需要安装 + + **如果您在安装过程中，发现存在兼容性问题，请您微信联系我们，我们将立马对兼容性问题进行修复。** + + **注意：Linkis本身不依赖Hive、Spark、python等服务，只有您需要用到hive、spark和python时，才需要在对应的节点进行安装。** + +### (2) 创建用户 + + 例如: **部署用户是hadoop账号** + +1. 在所有需要部署的机器上创建部署用户，用于安装 + + sudo useradd hadoop + + +2. 因为Linkis的服务是以 sudo -u ${linux-user} 方式来切换引擎，从而执行作业，所以部署用户需要有 sudo 权限，而且是免密的。 + + vi /etc/sudoers + + hadoop ALL=(ALL) NOPASSWD: NOPASSWD: ALL + +3. **如果您需要使用Spark和Hive，还需在每台安装节点设置如下的全局环境变量**， + + 修改安装用户的.bash_rc，命令如下： + + vim /home/hadoop/.bash_rc + + 下方为环境变量示例： + + export JAVA_HOME=/nemo/jdk1.8.0_141 + #HADOOP + export HADOOP_HOME=/appcom/Install/hadoop + export HADOOP_CONF_DIR=/appcom/config/hadoop-config + #Spark + export SPARK_HOME=/appcom/Install/spark + export SPARK_CONF_DIR=/appcom/config/spark-config/spark-submit + #Hive + export HIVE_HOME=/appcom/Install/hive + export HIVE_CONF_DIR=/appcom/config/hive-config + + +### (3) ssh免密配置 + + +在部署机器和其他安装机器上配置ssh免密登录，如果要在部署机上安装，需要将主机器和各个其它机器SSH打通 + + +## 2、编译打包（可跳过）： + + ### 如果用户不想自己编译，可以直接在release页面下载安装包，本步骤可以直接跳过。 + + 从git获取项目代码后，使用maven打包项目安装包。 + + （1） **如果您是本地第一次使用，必须在最外层工程pom.xml所在目录先执行以下命令**： + + mvn -N install + + （2） 在最外层工程pom.xml所在目录执行以下命令 + + mvn clean install + + （3） 获取安装包，在工程的assembly->target目录下： + + wedatasphere-linkis-x.x.x-dist.tar.gz + +## 3、安装： + + 先解压安装包到安装目录，并对解压后文件进行配置修改。 + + tar -xvf wedatasphere-linkis-x.x.x-dist.tar.gz + + （1）修改基础配置 + + vi /conf/config.sh + + ``` + deployUser=hadoop #指定部署用户 + LINKIS_INSTALL_HOME=/appcom/Install/Linkis # 指定安装目录 + WORKSPACE_USER_ROOT_PATH=/tmp/hadoop # 指定用户根目录，一般用于存储用户的脚本文件和日志文件等，是用户的工作空间。 + HDFS_USER_ROOT_PATH=hdfs:///tmp/linkis # 指定用户的HDFS根目录，一般用于存储Job的结果集文件 + + ``` + + （2）修改数据库配置 + + vi ./conf/db.sh + + ``` + # 设置数据库的连接信息 + # 包括IP地址、数据库名称、用户名、端口 + # 主要用于存储用户的自定义变量、配置参数、UDF和小函数，以及提供JobHistory的底层存储 + MYSQL_HOST= + MYSQL_PORT= + MYSQL_DB= + MYSQL_USER= + MYSQL_PASSWORD= + ``` + + (3) 导入相关表到数据库 + + 因为担心用户重复执行install.sh脚本，把数据库中的用户数据清空，所以在install.sh中注释掉了以下的两行导入表元数据代码。 + + 如果您确定是**第一次**部署，可以手动修改install.sh脚本，将注释去掉。 + + ```shell + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB -e "source ${workDir}/db/linkis_ddl.sql" + mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB -e "source ${workDir}/db/linkis_dml.sql" + ``` + + （4）执行安装脚本： + + sh ./bin/install.sh + +## 4、启动服务： + sh ./bin/start-all.sh + +## 5、查看服务 + 可以在Eureka界面查看服务启动成功情况，查看方法： + + 使用http://${EUREKA_INSTALL_IP}:${EUREKA_PORT}, 在浏览器中打开，查看服务是否注册成功。 + + 如果您没有在config.sh指定EUREKA_INSTALL_IP和EUREKA_INSTALL_IP，则HTTP地址为：http://127.0.0.1:20303 + + 如下图，如您的Eureka主页出现以下微服务，则表示服务都启动成功，可以正常对外提供服务了： + + ![Eureka](../images/ch1/Eureka_homepage.png) + diff --git "a/docs/zh_CN/ch2/linkis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md" "b/docs/zh_CN/ch2/linkis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md" new file mode 100644 index 00000000000..fe408ea4ce8 --- /dev/null +++ "b/docs/zh_CN/ch2/linkis\345\277\253\351\200\237\344\275\277\347\224\250\346\226\207\346\241\243.md" @@ -0,0 +1,155 @@ +### Linkis快速使用文档 +#### 1.概述 +Linkis为用户提供了Java客户端的实现，用户可以使用UJESClient对Linkis后台服务实现快速访问。 + +#### 2 快速运行 + + 我们在ujes/client/src/test模块下，提供了UJESClient的两个测试类： + + com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTestJ # 基于Java实现的测试类 + com.webank.wedatasphere.linkis.ujes.client.UJESClientImplTest # 基于Scala实现的测试类 + + 如果您clone了Linkis的源代码，可以直接运行这两个测试类。 + + **下面具体介绍如何快速实现一次对Linkis的代码提交执行。** + +#### 3 快速实现 + +##### 3.1 maven依赖 + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-client + 0.8.0 + +``` + +#### 3.2 参考实现 + +- **JAVA** +```java +package com.webank.bdp.dataworkcloud.ujes.client; + +import com.webank.wedatasphere.Linkis.common.utils.Utils; +import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy; +import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfig; +import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder; +import com.webank.wedatasphere.Linkis.ujes.client.UJESClient; +import com.webank.wedatasphere.Linkis.ujes.client.UJESClientImpl; +import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction; +import com.webank.wedatasphere.Linkis.ujes.client.request.ResultSetAction; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobExecuteResult; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobInfoResult; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobProgressResult; +import com.webank.wedatasphere.Linkis.ujes.client.response.JobStatusResult; +import org.apache.commons.io.IOUtils; + +import java.util.concurrent.TimeUnit; + + +public class UJESClientImplTestJ{ + public static void main(String[] args){ + // 1. 配置DWSClientBuilder，通过DWSClientBuilder获取一个DWSClientConfig + DWSClientConfig clientConfig = ((DWSClientConfigBuilder) (DWSClientConfigBuilder.newBuilder() + .addUJESServerUrl("http://${ip}:${port}") //指定ServerUrl，Linkis服务器端网关的地址,如http://{ip}:{port} + .connectionTimeout(30000) //connectionTimeOut 客户端连接超时时间 + .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //是否启用注册发现，如果启用，会自动发现新启动的Gateway + .loadbalancerEnabled(true) // 是否启用负载均衡，如果不启用注册发现，则负载均衡没有意义 + .maxConnectionSize(5) //指定最大连接数，即最大并发数 + .retryEnabled(false).readTimeout(30000) //执行失败，是否允许重试 + .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy Linkis认证方式 + .setAuthTokenKey("${username}").setAuthTokenValue("${password}"))) //认证key，一般为用户名; 认证value，一般为用户名对应的密码 + .setDWSVersion("v1").build(); //Linkis后台协议的版本，当前版本为v1 + + // 2. 通过DWSClientConfig获取一个UJESClient + UJESClient client = new UJESClientImpl(clientConfig); + + // 3. 开始执行代码 + JobExecuteResult jobExecuteResult = client.execute(JobExecuteAction.builder() + .setCreator("LinkisClient-Test") //creator，请求Linkis的客户端的系统名，用于做系统级隔离 + .addExecuteCode("show tables") //ExecutionCode 请求执行的代码 + .setEngineType(JobExecuteAction.EngineType$.MODULE$.HIVE()) // 希望请求的Linkis的执行引擎类型，如Spark hive等 + .setUser("johnnwang") //User，请求用户；用于做用户级多租户隔离 + .build()); + System.out.println("execId: " + jobExecuteResult.getExecID() + ", taskId: " + jobExecuteResult.taskID()); + + // 4. 获取脚本的执行状态 + JobStatusResult status = client.status(jobExecuteResult); + while(!status.isCompleted()) { + // 5. 获取脚本的执行进度 + JobProgressResult progress = client.progress(jobExecuteResult); + Utils.sleepQuietly(500); + status = client.status(jobExecuteResult); + } + + // 6. 获取脚本的Job信息 + JobInfoResult jobInfo = client.getJobInfo(jobExecuteResult); + // 7. 获取结果集列表（如果用户一次提交多个SQL，会产生多个结果集） + String resultSet = jobInfo.getResultSetList(client)[0]; + // 8. 通过一个结果集信息，获取具体的结果集 + Object fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser()).build()).getFileContent(); + System.out.println("fileContents: " + fileContents); + IOUtils.closeQuietly(client); + } +} +``` + +- **SCALA** +```scala + +import java.util.concurrent.TimeUnit + +import com.webank.wedatasphere.Linkis.common.utils.Utils +import com.webank.wedatasphere.Linkis.httpclient.dws.authentication.StaticAuthenticationStrategy +import com.webank.wedatasphere.Linkis.httpclient.dws.config.DWSClientConfigBuilder +import com.webank.wedatasphere.Linkis.ujes.client.request.JobExecuteAction.EngineType +import com.webank.wedatasphere.Linkis.ujes.client.request.{JobExecuteAction, ResultSetAction} +import org.apache.commons.io.IOUtils + +object UJESClientImplTest extends App { + + // 1. 配置DWSClientBuilder，通过DWSClientBuilder获取一个DWSClientConfig + val clientConfig = DWSClientConfigBuilder.newBuilder() + .addUJESServerUrl("http://${ip}:${port}") //指定ServerUrl，Linkis服务器端网关的地址,如http://{ip}:{port} + .connectionTimeout(30000) //connectionTimeOut 客户端连接超时时间 + .discoveryEnabled(true).discoveryFrequency(1, TimeUnit.MINUTES) //是否启用注册发现，如果启用，会自动发现新启动的Gateway + .loadbalancerEnabled(true) // 是否启用负载均衡，如果不启用注册发现，则负载均衡没有意义 + .maxConnectionSize(5) //指定最大连接数，即最大并发数 + .retryEnabled(false).readTimeout(30000) //执行失败，是否允许重试 + .setAuthenticationStrategy(new StaticAuthenticationStrategy()) //AuthenticationStrategy Linkis认证方式 + .setAuthTokenKey("${username}").setAuthTokenValue("${password}") //认证key，一般为用户名; 认证value，一般为用户名对应的密码 + .setDWSVersion("v1").build() //Linkis后台协议的版本，当前版本为v1 + + // 2. 通过DWSClientConfig获取一个UJESClient + val client = UJESClient(clientConfig) + + // 3. 开始执行代码 + val jobExecuteResult = client.execute(JobExecuteAction.builder() + .setCreator("LinkisClient-Test") //creator，请求Linkis的客户端的系统名，用于做系统级隔离 + .addExecuteCode("show tables") //ExecutionCode 请求执行的代码 + .setEngineType(EngineType.SPARK) // 希望请求的Linkis的执行引擎类型，如Spark hive等 + .setUser("${username}").build()) //User，请求用户；用于做用户级多租户隔离 + println("execId: " + jobExecuteResult.getExecID + ", taskId: " + jobExecuteResult.taskID) + + // 4. 获取脚本的执行状态 + var status = client.status(jobExecuteResult) + while(!status.isCompleted) { + // 5. 获取脚本的执行进度 + val progress = client.progress(jobExecuteResult) + val progressInfo = if(progress.getProgressInfo != null) progress.getProgressInfo.toList else List.empty + println("progress: " + progress.getProgress + ", progressInfo: " + progressInfo) + Utils.sleepQuietly(500) + status = client.status(jobExecuteResult) + } + + // 6. 获取脚本的Job信息 + val jobInfo = client.getJobInfo(jobExecuteResult) + // 7. 获取结果集列表（如果用户一次提交多个SQL，会产生多个结果集） + val resultSet = jobInfo.getResultSetList(client).head + // 8. 通过一个结果集信息，获取具体的结果集 + val fileContents = client.resultSet(ResultSetAction.builder().setPath(resultSet).setUser(jobExecuteResult.getUser).build()).getFileContent + println("fileContents: " + fileContents) + IOUtils.closeQuietly(client) +} +``` diff --git "a/docs/zh_CN/ch3/HTTP\346\216\245\345\205\245\346\226\207\346\241\243.md" "b/docs/zh_CN/ch3/HTTP\346\216\245\345\205\245\346\226\207\346\241\243.md" new file mode 100644 index 00000000000..534ff287a45 --- /dev/null +++ "b/docs/zh_CN/ch3/HTTP\346\216\245\345\205\245\346\226\207\346\241\243.md" @@ -0,0 +1,206 @@ +## Linkis HTTP接入文档 + + +#### 1.概述 +        Linkis提供了HTTP的接入方式，方便上层应用的前端，快速实现接入。 + + +#### 2.前端接入 +        注意事项：Linkis项目前端接口提供了两种方式，HTTP和WebSocket。Websocket方式相比于HTTP方式具有对服务器友好，信息推送更加及时等优势，但是WebSocket在用户使用的时候可能出现断开连接的情况。 + +数据开发IDE工具[Scriptis](https://github.com/WeBankFinTech/Scriptis)在对接Linkis时，采用了WebSocket和HTTP结合的方式，正常情况下使用websocket与Linkis进行通信，出现WebSocket断开连接的时候，就会切换为HTTP的方式与后台进行交互。 + +##### 2.1 接口规范 + +Linkis在前后端进行交互的时候，自定义了一套自己的接口规范。 + +**1).URL规范** +``` +/api/rest_j/v1/{applicationName}/.+ +/api/rest_s/v1/{applicationName}/.+ +``` + +- rest_j表示接口符合Jersey规范 +- rest_s表示接口符合springMVC Rest规范 +- v1为服务的版本号，**版本号会随着Linkis版本进行升级** +- {applicationName}为微服务名 + +**2).请求规范** + +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{}, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + +**3).响应规范** + +```json +{"method":"/api/rest_j/v1/entrance/execute","status":0, "message":"成功！","data":{}} +``` + +- method：返回请求的Restful API URL，主要是websocket模式需要使用。 +- status：返回状态信息，其中：-1表示没有登录，0表示成功，1表示错误，2表示验证失败，3表示没该接口的访问权限。 +- data：返回具体的数据。 +- message：返回请求的提示信息。如果status非0时，message返回的是错误信息，其中data有可能存在stack字段，返回具体的堆栈信息。 + +##### 2.2 HTTP接口描述 +HTTP接口需要在提交执行之后,需要采用HTTP轮询的方式请求获取作业的状态、日志、进度等信息 + + + +**1).请求执行** + +- 接口 `/api/rest_j/entrance/execute` + +- 提交方式 `POST` + +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{ + "params": { + "variable":{ + "k1":"v1" + }, + "configuration":{ + "special":{ + "k2":"v2" + }, + "runtime":{ + "k3":"v3" + }, + "startup":{ + "k4":"v4" + } + } + }, + "executeApplicationName":"spark", + "executionCode":"show tables", + "runType":"sql", + "source":{ + "scriptPath": "/home/Linkis/Linkis.sql" + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" + } +} +``` +- 请求体data中的参数描述如下 + + +| 参数名 | 参数定义 | 类型 | 备注 | +| ------------ | ------------ | ------------ | ------------ | +| executeApplicationName | 用户所期望使用的引擎服务，如Spark、hive等| String | 不可为空 | +| requestApplicationName | 发起请求的系统名 | String | 可以为空 | +| params | 用户指定的运行服务程序的参数 | Map | 必填，里面的值可以为空 | +| executionCode | 用户提交的执行代码 | String |不可为空 | +| runType | 当用户执行如spark服务时，可以选择python、R、SQL等runType| String | 不可为空 | +| scriptPath | 用户提交代码脚本的存放路径 | String | 如果是IDE的话，与executionCode不能同时为空 | + 表1 请求体参数描述 + +- 返回示例 +```json +{ + "method": "/api/rest_j/v1/entrance/execute", + "status": 0, + "message": "请求执行成功", + "data": { + "execID": "030418IDEhivebdpdwc010004:10087IDE_johnnwang_21", + "taskID": "123" + } +} +``` + +- execID是用户任务提交到UJES之后，为该任务生成的唯一标识的执行ID，为String类型，这个ID只在任务运行时有用，类似PID的概念。ExecID的设计为(requestApplicationName长度)(executeAppName长度1)(Instance长度2)${requestApplicationName}${executeApplicationName}${entranceInstance信息ip+port}${requestApplicationName}_${umUser}_${index} +- taskID 是表示用户提交task的唯一ID，这个ID由数据库自增生成，为Long 类型 + + +**2).获取状态**
+ +- 接口 `/api/rest_j/entrance/${execID}/status` +- 提交方式 `GET`
+- 返回示例 + +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/status", + "status": 0, + "message": "获取状态成功", + "data": { + "execID": "${execID}", + "status": "Running" + } +} +``` + +**3).获取日志**
+ +- 接口 `/api/rest_j/entrance/${execID}/log?fromLine=${fromLine}&size=${size}` +- 提交方式 `GET` +- 请求参数fromLine是指从第几行开始获取，size是指该次请求获取几行日志 +- 返回示例，其中返回的fromLine需要下次日志请求的参数 + +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "返回日志信息", + "data": { + "execID": "${execID}", + "log": ["error日志","warn日志","info日志", "all日志"], + "fromLine": 56 + } +} +``` + +**4).获取进度**
+ +- 接口 `/api/rest_j/entrance/${execID}/progress` +- 提交方式 `GET`
+- 返回示例 + +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/progress", + "status": 0, + "message": "返回进度信息", + "data": { + "execID": "${execID}", + "progress": 0.2, + "progressInfo": [ + { + "id": "job-1", + "succeedTasks": 2, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + }, + { + "id": "job-2", + "succeedTasks": 5, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + } + ] + } +} +``` +**5).kill任务**
+ +- 接口 `/api/rest_j/entrance/${execID}/kill` +- 提交方式 `POST` +- 返回示例，其中返回的fromLine需要下次日志请求的参数 + +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/kill", + "status": 0, + "message": "OK", + "data": { + "execID":"${execID}" + } +} +``` \ No newline at end of file diff --git "a/docs/zh_CN/ch3/Linkis\345\246\202\344\275\225\346\216\245\345\205\245\346\226\260\347\232\204\345\272\225\345\261\202\350\256\241\347\256\227\345\255\230\345\202\250\345\274\225\346\223\216.md" "b/docs/zh_CN/ch3/Linkis\345\246\202\344\275\225\346\216\245\345\205\245\346\226\260\347\232\204\345\272\225\345\261\202\350\256\241\347\256\227\345\255\230\345\202\250\345\274\225\346\223\216.md" new file mode 100644 index 00000000000..8d14e2cc6d9 --- /dev/null +++ "b/docs/zh_CN/ch3/Linkis\345\246\202\344\275\225\346\216\245\345\205\245\346\226\260\347\232\204\345\272\225\345\261\202\350\256\241\347\256\227\345\255\230\345\202\250\345\274\225\346\223\216.md" @@ -0,0 +1,100 @@ +# Linkis如何接入新的底层计算存储引擎 + +## 1. 前言 + +开发人员在使用Linkis时，不但可以直接使用Linkis已经开发的执行引擎如：Spark、Hive、Python等，也可以根据自己的需要，开发实现自己的新计算存储引擎。 + +Linkis接入新计算存储引擎的方式非常简单，可以分成Entrance，EngineManager和Engine几个模块。 + +其中Entrance、EngineManager和Engine三个模块的作用和架构可以查看[UJES架构设计文档](../ch4/Linkis-UJES设计文档.md)。 + +## 2. 申明 + +Linkis底层使用了Spring框架，所以必须遵循Spring开发规范。 + +Linkis的底层架构非常灵活，所有的顶层接口几乎都提供了一些通用实现，如果用户想要使用自己编写的类，可以直接依赖注入覆盖掉通用实现。 + +### 2.1 Entrance模块接入 + +**1) maven依赖** + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-entrance + 0.8.0 + +``` + +**2) 必须要实现的接口** + +新的Entrance没有必须要实例化的接口，以下接口可以根据需要进行定制化实现。 + +- EntranceParser。用于将前端传递过来的一个请求Map，转换成一个可被持久化的Task。该类已提供了AbstractEntranceParser，用户只需实现parseToTask方法即可，系统默认提供了CommonEntranceParser实现。 +- EngineRequester。用于获得一个RequestEngine类，该类用于向EngineManager请求一个新的Engine。 +- Scheduler。用于实现调度，默认已实现了多用户并发、单个用户内FIFO执行的调度模式，如无特殊需求，不建议实例化。 + +### 2.2 EngineManager模块接入 + +**1) maven依赖** + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-enginemanager + 0.8.0 + +``` + +**2) 需要实现的接口** + +新的EngineManager必须实现以下接口: + +- EngineCreator，已存在AbstractEngineCreator，需实现createProcessEngineBuilder方法，用于创建一个EngineBuilder。 + + ProcessEngineBuilder已默认提供了一个JavaProcessEngineBuilder类，这个类是一个abstract类，已默认将必要的classpath、JavaOpts、GC文件路径、日志文件路径，以及测试模式下DEBUG端口的开启已做好了。 + + 如果创建的Engine是一个Java进程，建议继承JavaJavaProcessEngineBuilder，只需要加入新引擎额外的classpath和JavaOpts即可。 + +- EngineResourceFactory，已存在AbstractEngineResourceFactory，需实现getRequestResource方法，用于拿到用户的个性化资源请求。 + +- resources，这是一个spring实体bean，主要用于向RM注册资源，resources是ModuleInfo的实例，需要用户提供一个，以供依赖注入。 + +根据需要进行实现： + +- hooks，这是一个spring实体bean，主要用于在创建并启动Engine的前后，加前置和后置hook。 + + 如果想使新的EngineHook生效，需要创建一个EngineHook[] hooks的spring bean，具体请参考com.webank.wedatasphere.linkis.enginemanager.impl.EngineManagerSpringConfiguration。 + +### 2.3 Engine模块接入 + +**1) maven依赖** + +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-engine + 0.8.0 + +``` + +**2)需要实现的接口** + + +Engine必须实现的接口如下： + +- EngineExecutorFactory。用于创建一个EngineExecutor，需实现createExecutor方法，具体为通过一个Map，Map会传入所有环境变量、用户引擎启动参数，用于创建一个EngineExecutor。 +- EngineExecutor。实际真正的执行器，用于提交执行entrance提交过来的代码。 + + 需要实现以下方法： + + 1. getActualUsedResources（该engine实际使用的资源） + 2. executeLine（执行一行通过CodeParser解析过的代码） + 3. executeCompletely（executeLine的补充方法，如果调用executeLine返回的是ExecuteIncomplete，这时会将新的Code和之前返回ExecuteIncomplete的代码同时传递给engine执行） + + +Engine非必须实现的接口或bean如下: + +- engineHooks: Array[EngineHook]，是一个spring bean。EngineHook是engine创建的前置和后置hook，目前系统已提供了2个hook：CodeGeneratorEngineHook用于加载UDF和函数，ReleaseEngineHook用于释放空闲的engine，如果不指定，系统默认会提供engineHooks=Array(ReleaseEngineHook) +- CodeParser。用于解析代码，以便一行一行执行。如果不指定，系统默认提供一个直接返回所有代码的CodeParser。 +- EngineParser，用于将一个RequestTask转换成可提交给Scheduler的Job，如果没有指定，系统会默认提供一个将RequestTask转换成CommonEngineJob的EngineParser。 diff --git "a/docs/zh_CN/ch3/WebSocket\346\216\245\345\205\245\346\226\207\346\241\243.md" "b/docs/zh_CN/ch3/WebSocket\346\216\245\345\205\245\346\226\207\346\241\243.md" new file mode 100644 index 00000000000..f572dc2e0e1 --- /dev/null +++ "b/docs/zh_CN/ch3/WebSocket\346\216\245\345\205\245\346\226\207\346\241\243.md" @@ -0,0 +1,204 @@ +## Linkis HTTP接入文档 + + +#### 1.概述 +        Linkis提供了HTTP的接入方式，方便上层应用的前端，快速实现接入。 + + +#### 2.前端接入 +        注意事项：Linkis项目前端接口提供了两种方式，HTTP和WebSocket。Websocket方式相比于HTTP方式具有对服务器友好，信息推送更加及时等优势，但是WebSocket在用户使用的时候可能出现断开连接的情况。 + +数据开发IDE工具[Scriptis](https://github.com/WeBankFinTech/Scriptis)在对接Linkis时，采用了WebSocket和HTTP结合的方式，正常情况下使用websocket与Linkis进行通信，出现WebSocket断开连接的时候，就会切换为HTTP的方式与后台进行交互。 + +##### 2.1 接口规范 + +Linkis在前后端进行交互的时候，自定义了一套自己的接口规范。 + +**1).URL规范** +``` +/api/rest_j/v1/{applicationName}/.+ +/api/rest_s/v1/{applicationName}/.+ +``` + +- rest_j表示接口符合Jersey规范 +- rest_s表示接口符合springMVC Rest规范 +- v1为服务的版本号，**版本号会随着Linkis版本进行升级** +- {applicationName}为微服务名 + +**2).请求规范** + +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{}, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + +**3).响应规范** + +```json +{"method":"/api/rest_j/v1/entrance/execute","status":0, "message":"成功！","data":{}} +``` + +- method：返回请求的Restful API URL，主要是websocket模式需要使用。 +- status：返回状态信息，其中：-1表示没有登录，0表示成功，1表示错误，2表示验证失败，3表示没该接口的访问权限。 +- data：返回具体的数据。 +- message：返回请求的提示信息。如果status非0时，message返回的是错误信息，其中data有可能存在stack字段，返回具体的堆栈信息。 + + +##### 2.2 WebSocket接口描述 + + +**1).建立连接** + +此接口是为了和Linkis建立一个WebSocket连接。 + +- `/api/rest_j/entrance/connect` +- 请求方式 **GET** +- 响应状态码 **101**
+ +**2).请求执行任务** + + +请求执行任务是将用户的作业提交到Linkis进行执行的接口 + +- 接口 `/api/rest_j/entrance/execute` +- 提交方式 `POST`
+- 请求JSON示例 + +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{ + "params": { + "variable":{ + "k1":"v1" + }, + "configuration":{ + "special":{ + "k2":"v2" + }, + "runtime":{ + "k3":"v3" + }, + "startup":{ + "k4":"v4" + } + } + }, + "executeApplicationName":"spark", + "executionCode":"show tables", + "runType":"sql", + "source":{ + "scriptPath": "/home/Linkis/Linkis.sql" + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" + } +} +``` + +- 请求体data中的参数描述如下 + + +| 参数名 | 参数定义 | 类型 | 备注 | +| ------------ | ------------ | ------------ | ------------ | +| executeApplicationName | 用户所期望使用的引擎服务，如Spark、hive等| String | 不可为空 | +| requestApplicationName | 发起请求的系统名 | String | 可以为空 | +| params | 用户指定的运行服务程序的参数 | Map | 必填，里面的值可以为空 | +| executionCode | 用户提交的执行代码 | String |不可为空 | +| runType | 当用户执行如spark服务时，可以选择python、R、SQL等runType| String | 不可为空 | +| scriptPath | 用户提交代码脚本的存放路径 | String | 如果是IDE的话，与executionCode不能同时为空 | + 表1 请求体参数描述 + +- 返回示例 + +```json +{ + "method": "/api/rest_j/v1/entrance/execute", + "status": 0, + "message": "请求执行成功", + "data": { + "execID": "030418IDEhivebdpdwc010004:10087IDE_johnnwang_21", + "taskID": "123" + } +} +``` + +- execID是用户任务提交到UJES之后，为该任务生成的唯一标识的执行ID，为String类型，这个ID只在任务运行时有用，类似PID的概念。ExecID的设计为(requestApplicationName长度)(executeAppName长度1)(Instance长度2)${requestApplicationName}${executeApplicationName}${entranceInstance信息ip+port}${requestApplicationName}_${umUser}_${index} +- taskID 是表示用户提交task的唯一ID，这个ID由数据库自增生成，为Long 类型 + + +**3).任务状态、日志、进度主动推送** + + +提交执行之后，任务的状态、日志、进度等信息都会由服务器主动推送，用websocket方式去主动进行请求。 + +请求的接口和下文中的HTTP是保持一致的，唯一不一样的是，websocket的请求shema是ws://,而HTTP的请求schema是http://。 + +WebSocket中的接口返回的示例如下 + +- 日志 + +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "返回日志信息", + "data": { + "execID": "${execID}", + "log": ["error日志","warn日志","info日志", "all日志"], + "taskID":28594, + "fromLine": 56 + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + +- 状态 + +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/status", + "status": 0, + "message": "返回状态信息", + "data": { + "execID": "${execID}", + "taskID":28594, + "status": "Running", + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + +- 进度 + +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "返回进度信息信息", + "data": { + "execID": "${execID}", + "taskID":28594, + "progress": 0.2, + "progressInfo": [ + { + "id": "job-1", + "succeedTasks": 2, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + }, + { + "id": "job-2", + "succeedTasks": 5, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + } + ] + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` \ No newline at end of file diff --git "a/docs/zh_CN/ch3/linkis\344\275\277\347\224\250\346\226\207\346\241\243.md" "b/docs/zh_CN/ch3/linkis\344\275\277\347\224\250\346\226\207\346\241\243.md" new file mode 100644 index 00000000000..fd4d170ba4b --- /dev/null +++ "b/docs/zh_CN/ch3/linkis\344\275\277\347\224\250\346\226\207\346\241\243.md" @@ -0,0 +1,363 @@ +## Linkis使用文档 + + +#### 1.概述 +        Linkis项目是微众银行大数据平台自研的一种大数据作业提交方案及可扩展框架。项目使用方式方便，用户可以结合微众银行另一项开源项目——意书(这里要用http链接)直接进行使用，当然用户也可以根据规定的前端接口进行接入。Linkis也提供了客户端的实现，用户可以通过使用Linkis的sdk直接访问服务端。另外，Linkis作为一个可扩展性很强的框架,用户可以通过SDK的方式开发自己的应用。 + + +#### 2.前端接入 +        Linkis项目前端接口提供了两种方式，HTTP和WebSocket。Websocket方式相比于HTTP方式具有对服务器友好，信息推送更加及时等优势，但是WebSocket在用户使用的时候可能出现断开连接的情况，所以开源项目意书在对接Linkis时候，采用了WebSocket和HTTP结合的方式，正常情况下使用websocket与Linkis进行通信，出现WebSocket断开连接的时候，就会切换为HTTP的方式与后台进行交互。 +##### 2.1接口规范 +Linkis在前后端进行交互的时候，自定义了一套自己的接口规范。
+**1).URL规范** +``` +/api/rest_j/v1/{applicationName}/.+ +/api/rest_s/v1/{applicationName}/.+ +``` + +- rest_j表示接口符合Jersey规范 +- rest_s表示接口符合springMVC Rest规范 +- v1为服务的版本号，**版本号会随着Linkis版本进行升级** +- {applicationName}为微服务名 + +**2).请求规范** +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{}, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + +**3).响应规范** +```json +{"method":"/api/rest_j/v1/entrance/execute","status":0, "message":"成功！","data":{}} +``` +- method：返回请求的Restful API URL，主要是websocket模式需要使用。 +- status：返回状态信息，其中：-1表示没有登录，0表示成功，1表示错误，2表示验证失败，3表示没该接口的访问权限。 +- data：返回具体的数据。 +- message：返回请求的提示信息。如果status非0时，message返回的是错误信息，其中data有可能存在stack字段，返回具体的堆栈信息。 + + +##### 2.2WebSocket接口描述 + + +**1).建立连接**
+
+此接口是为了和Linkis建立一个WebSocket连接。 +- `/api/rest_j/entrance/connect` +- 请求方式 **GET** +- 响应状态码 **101**
+ +**2).请求执行任务**
+
+请求执行任务是将用户的作业提交到Linkis进行执行的接口 +- 接口 `/api/rest_j/entrance/execute` +- 提交方式 `POST`
+- 请求JSON示例 +```json +{ + "method":"/api/rest_j/v1/entrance/execute", + "data":{ + "params": { + "variable":{ + "k1":"v1" + }, + "configuration":{ + "special":{ + "k2":"v2" + }, + "runtime":{ + "k3":"v3" + }, + "startup":{ + "k4":"v4" + } + } + }, + "executeApplicationName":"spark", + "executionCode":"show tables", + "runType":"sql", + "source":{ + "scriptPath": "/home/Linkis/Linkis.sql" + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" + } +} +``` +- 请求体data中的参数描述如下
+ +| 参数名 | 参数定义 | 类型 | 备注 | +| ------------ | ------------ | ------------ | ------------ | +| executeApplicationName | 用户所期望使用的引擎服务，如Spark、hive等| String | 不可为空 | +| requestApplicationName | 发起请求的系统名 | String | 可以为空 | +| params | 用户指定的运行服务程序的参数 | Map | 必填，里面的值可以为空 | +| executionCode | 用户提交的执行代码 | String |不可为空 | +| runType | 当用户执行如spark服务时，可以选择python、R、SQL等runType| String | 不可为空 | +| scriptPath | 用户提交代码脚本的存放路径 | String | 如果是IDE的话，与executionCode不能同时为空 | + 表1 请求体参数描述 + +- 返回示例 +```json +{ + "method": "/api/rest_j/v1/entrance/execute", + "status": 0, + "message": "请求执行成功", + "data": { + "execID": "030418IDEhivebdpdwc010004:10087IDE_johnnwang_21", + "taskID": "123" + } +} +``` +- execID是用户任务提交到UJES之后，为该任务生成的唯一标识的执行ID，为String类型，这个ID只在任务运行时有用，类似PID的概念。ExecID的设计为(requestApplicationName长度)(executeAppName长度1)(Instance长度2)${requestApplicationName}${executeApplicationName}${entranceInstance信息ip+port}${requestApplicationName}_${umUser}_${index} +- taskID 是表示用户提交task的唯一ID，这个ID由数据库自增生成，为Long 类型 + +**3).任务状态、日志、进度主动推送**
+ +提交执行之后，任务的状态、日志、进度等信息都会由服务器主动推送，用websocket方式去主动进行请求。 +请求的接口和下文中的HTTP是保持一致的，唯一不一样的是，websocket的请求shema是ws://,而HTTP的请求schema是http://。 + +WebSocket中的接口返回的示例如下 +- 日志 +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "返回日志信息", + "data": { + "execID": "${execID}", + "log": ["error日志","warn日志","info日志", "all日志"], + "taskID":28594, + "fromLine": 56 + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` +- 状态 +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/status", + "status": 0, + "message": "返回状态信息", + "data": { + "execID": "${execID}", + "taskID":28594, + "status": "Running", + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` +- 进度 +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "返回进度信息信息", + "data": { + "execID": "${execID}", + "taskID":28594, + "progress": 0.2, + "progressInfo": [ + { + "id": "job-1", + "succeedTasks": 2, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + }, + { + "id": "job-2", + "succeedTasks": 5, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + } + ] + }, + "websocketTag":"37fcbd8b762d465a0c870684a0261c6e" +} +``` + + + +##### 2.3HTTP接口描述 +HTTP接口需要在提交执行之后,需要采用HTTP轮询的方式请求获取作业的状态、日志、进度等信息 + + + +**1).请求执行** + +- 接口 `/api/rest_j/entrance/execute` +- 提交方式 `POST`
+- 请求JSON示例，请求的参数描述与表1一致 +```json +{ + "params": {}, + "executeApplicationName":"spark", + "executionCode":"show tables", + "runType":"sql", + "source":{ + "scriptPath": "/home/Linkis/Linkis.sql" + } +} +``` +- 响应返回与websocket返回一致，都会获取一个execID和taskID + +**2).获取状态**
+
+建立连接是WebSocket特有的一个接口,是为了和Linkis建立一个WebSocket连接 +- 接口 `/api/rest_j/entrance/${execID}/status` +- 提交方式 `GET`
+- 返回示例 +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/status", + "status": 0, + "message": "获取状态成功", + "data": { + "execID": "${execID}", + "status": "Running" + } +} +``` + +**3).获取日志**
+
+建立连接是WebSocket特有的一个接口,是为了和Linkis建立一个WebSocket连接 +- 接口 `/api/rest_j/entrance/${execID}/log?fromLine=${fromLine}&size=${size}` +- 提交方式 `GET` +- 请求参数fromLine是指从第几行开始获取，size是指该次请求获取几行日志 +- 返回示例，其中返回的fromLine需要下次日志请求的参数 +```json +{ + "method": "/api/rest_j/v1/entrance/${execID}/log", + "status": 0, + "message": "返回日志信息", + "data": { + "execID": "${execID}", + "log": ["error日志","warn日志","info日志", "all日志"], + "fromLine": 56 + } +} +``` + +**4).获取进度**
+
+建立连接是WebSocket特有的一个接口,是为了和Linkis建立一个WebSocket连接 +- 接口 `/api/rest_j/entrance/${execID}/progress` +- 提交方式 `GET`
+- 返回示例 +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/progress", + "status": 0, + "message": "返回进度信息", + "data": { + "execID": "${execID}", + "progress": 0.2, + "progressInfo": [ + { + "id": "job-1", + "succeedTasks": 2, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + }, + { + "id": "job-2", + "succeedTasks": 5, + "failedTasks": 0, + "runningTasks": 5, + "totalTasks": 10 + } + ] + } +} +``` +**5).kill任务**
+
+建立连接是WebSocket特有的一个接口,是为了和Linkis建立一个WebSocket连接 +- 接口 `/api/rest_j/entrance/${execID}/kill` +- 提交方式 `POST` +- 返回示例，其中返回的fromLine需要下次日志请求的参数 +```json +{ + "method": "/api/rest_j/v1/entrance/{execID}/kill", + "status": 0, + "message": "OK", + "data": { + "execID":"${execID}" + } +} +``` + + + +### 3.客户端SDK接入 +客户端SDK接入请查看 +* [Linkis快速使用文档](/docs/zh_CH/ch2/linkis快速使用文档.md) + + + +#### 4. 多引擎类型支持 +后台开发人员在使用Linkis的时候，不但可以直接使用Linkis已经开发的执行引擎，也可以根据自己的需求使用框架开发出自己的应用。Linkis的接入方式简单,可以分成Entrance，EngineManager和Engine几个模块。其中Entrance、EngineManager和Engine三个模块的作用和架构可以查看UJES架构设计文档(**真实链接**) + +#### 约定 + +Linkis项目使用了Spring框架作为底层技术，所以一些类实例可以直接通过Spring的注解进行注入。Linkis框架提供了一些通用实现，如果用户想要使用自己编写的类，可以使用直接使用并覆盖掉通用实现。 + +##### 4.1Entrance模块接入 +**1)maven依赖** +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-entrance + 0.8.0 + +``` +**2)需要实现的接口** + +Entrance没有必须要实例化的接口，以下接口可以根据需要进行实现。 +- EntranceParser。用于将前端传递过来的一个请求Map，转换成一个可被持久化的Task。该类已提供了AbstractEntranceParser，用户只需实现parseToTask方法即可，系统默认提供了CommonEntranceParser实现。 +- EngineRequester。用于获得一个RequestEngine类，该类用于向EngineManager请求一个新的Engine。 +- Scheduler。用于实现调度，默认已实现了多用户并发、单个用户内FIFO执行的调度模式，如无特殊需求，不建议实例化。 + +##### 4.2EngineManager模块接入 +**1)maven依赖** +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-enginemanager + 0.8.0 + +``` + +**2)需要实现的接口** + +EngineManager需要对以下接口根据需要进行实现: +- EngineCreator，已存在AbstractEngineCreator，需实现createProcessEngineBuilder方法，用于创建一个EngineBuilder。 +在这里，ProcessEngineBuilder已默认提供了一个JavaProcessEngineBuilder类，这个类是一个abstract类，已默认将必要的classpath、JavaOpts、GC文件路径、日志文件路径，以及测试模式下DEBUG端口的开启已做好了。现JavaProcessEngineBuilder，只需要加入额外的classpath和JavaOpts即可。 +- EngineResourceFactory，已存在AbstractEngineResourceFactory，需实现getRequestResource方法，用于拿到用户的个性化资源请求。 +- hooks，这是一个spring实体bean，主要用于在创建并启动Engine的前后，加前置和后置hook，需要用户提供一个Array[EngineHook]，以供依赖注入。 +- resources，这是一个spring实体bean，主要用于像RM注册资源，resources是ModuleInfo的实例，需要用户提供一个，以供依赖注入。 + + +##### 4.3Engine模块接入 +**1)maven依赖** +```xml + + com.webank.wedatasphere.Linkis + Linkis-ujes-engine + 0.8.0 + +``` +**2)需要实现的接口** + +Engine必须实现的接口如下： +- EngineExecutorFactory。用于创建一个EngineExecutor，需实现createExecutor方法，具体为通过一个Map，创建一个EngineExecutor。 +- EngineExecutor。实际真正的执行器，用于提交执行entrance提交过来的代码。需要实现getActualUsedResources（该engine实际使用的资源）、executeLine（执行一行通过CodeParser解析过的代码）、executeCompletely（executeLine的补充方法，如果调用executeLine返回的是ExecuteIncomplete，这时会将新的Code和之前返回ExecuteIncomplete的代码同时传递给engine执行） + +Engine非必须实现的接口或bean如下: +- engineHooks: Array[EngineHook]，是一个spring bean。EngineHook是engine创建的前置和后置hook，目前系统已提供了2个hook：CodeGeneratorEngineHook用于加载UDF和函数，ReleaseEngineHook用于释放空闲的engine，如果不指定，系统默认会提供engineHooks=Array(ReleaseEngineHook) +- CodeParser。用于解析代码，以便一行一行执行。如果不指定，系统默认提供一个直接返回所有代码的CodeParser。 +- EngineParser，用于将一个RequestTask转换成可提交给Scheduler的Job，如果没有指定，系统会默认提供一个将RequestTask转换成CommonEngineJob的EngineParser。 diff --git "a/docs/zh_CN/ch3/linkis\350\257\246\347\273\206\344\273\213\347\273\215\346\226\207\346\241\243.md" "b/docs/zh_CN/ch3/linkis\350\257\246\347\273\206\344\273\213\347\273\215\346\226\207\346\241\243.md" new file mode 100644 index 00000000000..eaf99d08c36 --- /dev/null +++ "b/docs/zh_CN/ch3/linkis\350\257\246\347\273\206\344\273\213\347\273\215\346\226\207\346\241\243.md" @@ -0,0 +1,61 @@ +# Linkis介绍文档 + +## 1. 概述 + +Linkis是微众银行大数据平台套件[WeDataSphere](https://github.com/WeBankFinTech/WeDataSphere)的数据中间件。 + +---- + +## 2. 背景 + +随着大数据技术的广泛应用，开源社区催生出层出不穷的上层应用和下层计算引擎。 + +通过引入多个开源组件来满足不同的业务需求，不断更新和丰富大数据平台架构，几乎是现阶段所有企业的通用做法。 + +如下图所示，当我们的上层应用、工具系统，和底层的计算存储组件越来越多时，整个数据平台的情况就会变成如上图的网状结构。 +
+ +![原大数据生态图](../images/ch3/原大数据生态图.jpg) + +
+不断引入新开源组件来解决痛点的同时，新的组件都要去对接不同的上层功能工具或底层计算存储引擎，越来越多的痛点也随之产生： + +1. 业务需求变化多端，而上层开源组件各具特色，用户使用起来割裂感强烈，且学习成本高昂。 +2. 大数据种类繁多，存储、计算起来非常复杂，而底层的存储和计算引擎层出不穷，开发人员必须具备完善的技术栈。 +3. 新开源组件的引入必然在多租户隔离、用户资源管理、用户开发管理等方面无法兼容原有数据平台，自上而下的定制化开发工作，不仅工程量大，而且存在重复造轮子现象，开发周期长。 +4. 很多数据平台的上层应用直接对接了底层的计算、存储引擎，一旦出现异常，问题千奇百怪，用户体验非常差，也没有办法进行运维沉淀。 + +---- + +## 3. 我们的探索 + +构建企业级大数据平台，提供丰富且强大的功能工具、数据应用系统，通常需要考虑如下的几个问题： +
+ +![原大数据生态图](../images/ch3/要解决的问题.jpg) +
+ +1. 如何快速具备多租户隔离、弹性扩缩容和资源管控等企业级能力？ +2. 如何复用已有的平台层能力，新引入的组件能快速扩展对接？ +2. 如何简化上层应用的接入，提供标准快捷的接入方式，不用再去引入一堆Hadoop、Hive和Spark的maven依赖，解决一大堆环境和参数问题？ +3. 如何解决应用孤岛和数据孤岛问题？在IDE开发工具开发的代码如何快速发布调度？数据血缘如何做到一站式全覆盖？ + +我们重新定义了数据平台的“连接”层的问题，同时基于以上的痛点，给出了我们独有的解决方案。 + +---- + +### 4. 解决方案 + +Linkis，一个打通了多个计算存储引擎（如：Spark、Flink、Hive、Python和HBase等），对外提供统一REST/WebSocket/JDBC接口，提供提交执行SQL、Pyspark、HiveQL、Scala等脚本能力的数据平台中间件。 + + +![原大数据生态图](../images/ch3/解决方案.jpg) + +
+Linkis提供以下几个通用的模块组件，用于解决抽象和拆解几乎绝大部分的大数据平台层问题： +
+ +![原大数据生态图](../images/ch3/功能模块图.png) + + +Linkis，构建金融级和企业级大数据平台的利器！ \ No newline at end of file diff --git "a/docs/zh_CN/ch4/Linkis-RM\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/zh_CN/ch4/Linkis-RM\350\256\276\350\256\241\346\226\207\346\241\243.md" new file mode 100644 index 00000000000..c4ee732ecd1 --- /dev/null +++ "b/docs/zh_CN/ch4/Linkis-RM\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -0,0 +1,158 @@ +# Linkis RM设计文档 +## 背景 +在微服务场景下，各种服务所需要消耗和占用的资源具有多样性，不像传统大型应用一样容易管理。Linkis RM提供对资源的统一分配和回收的服务，在大量服务高频率启动和关闭的情况下，保证服务对资源的消耗不超出限制。 +## 产品设计 +### 总体架构图 +RM维护引擎管理器上报的可用资源信息，处理引擎提出的资源申请，并记录成功申请后的实际资源使用信息。 + +1. 引擎管理器，简称EM：处理启动引擎请求的微服务。EM作为资源的提供者，负责向RM注册资源(register)和下线资源(unregister)。同时，EM作为引擎的管理者，负责代替引擎向RM申请资源。每一个EM实例，均在RM中有一条对应的资源记录，包含它提供的总资源、保护资源等信息，并动态更新已使用资源。 + +1. 引擎，Engine，又称应用：执行用户作业的微服务。同时，引擎作为资源的实际使用者，负责向RM上报实际使用资源和释放资源。每一个Engine，均在RM中有一条对应的资源记录：在启动过程中，体现为锁定资源；在运行过程中，体现为已使用资源；在被结束之后，该资源记录随之被删除。 + +### 数据库表结构设计 +``` + +用户资源记录表： +linkis_user_resource_meta_data: + id + user + ticket_id + creator + em_application_name + em_instance + engine_application_name + engine_instance + user_locked_resource: 直接存储json + user_used_resource: json + resource_type + locked_time + used_time + +模块资源记录表： +linkis_em_resource_meta_data: + id + em_application_name + em_instance + total_resource:json + protected_resource:json + resource_policy + used_resource:json + left_resource:json + locked_resource:json + register_time: long + +模块policy表： +linkis_em_meta_data: + id + em_name + resource_request_policy + +锁：该表需要添加unique constraint：（scope，user, module_application_name, module_instance），用来保证锁不被强制多次同时获取。 +linkis_resource_lock: + id + user + em_application_name + em_instance + +``` + +### 资源的类型与格式 + +如上图所示，所有的资源类均实现一个顶层的Resource接口，该接口定义了所有资源类均需要支持的计算和比较的方法，并进行相应的数学运算符的重载，使得资源之间能够像数字一样直接被计算和比较。 + +|运算符|对应方法|运算符|对应方法| +|:---- |:---|:----- |:----- | +|+ |add |> |moreThan | +|- |minus |< | lessThan | +|* |multiply |= | equals | +| / |divide |>= | notLessThan | +| <= |notMoreThan | + + +当前支持的资源类型如下表所示，所有的资源都有对应的json序列化与反序列化方法，能够通过json格式进行存储和在网络间传递： + +|资源类型| 描述| +|:---- |:---| +|MemoryResource |MemoryResource | +|CPUResource |CPU资源| +|LoadResource |同时具备内存与CPU的资源| +|YarnResource |Yarn队列资源（队列，队列内存，队列CPU，队列实例数）| +|LoadInstanceResource |服务器资源（内存，CPU，实例数）| +|DriverAndYarnResource |驱动器与执行器资源（同时具备服务器资源，Yarn队列资源）| +|SpecialResource |其它自定义资源| + + +## 记录EM上报的可用资源 + +1. 持有资源的EM在启动时，将通过RPC调用register接口，传入json格式的资源来进行资源注册。需要向register接口提供的参数如下： + 1) 总资源：该EM能够提供的资源总数。 + 2) 保护资源：当剩余资源小于该资源时，不再允许继续分配资源。 + 3) 资源类型：如LoadResource，DriverAndYarnResource等类型名称。 + 4) EM名称：如sparkEngineManager等提供资源的EM名称。 + 5) EM实例：机器名加端口名。 +1. RM在收到资源注册请求后，在表linkis_module_resource_meta_data中新增一条记录，内容与接口的参数信息一致。 +1. 持有资源的EM在关闭时，将通过RPC调用unregister接口，传入自己的EM实例信息作为参数，来进行资源的下线。 +1. RM在收到资源下线请求后，在linkis_module_resource_meta_data表中找到EM实例信息对应的那一行，进行删除处理；同时在linkis_user_resource_meta_data表中，找到该EM实例对应的所有行，进行删除处理。 + +## 资源的分配与回收 +1. 接收用户的资源申请。 + a) RM提供requestResource接口给EM上报资源申请，该接口接受EM实例、用户、Creator和Resource对象作为参数。requestResource接受一个可选的时间参数，当处理事件超出该时间参数的限制时，该资源申请将自动作为失败处理。 +2. 判断是否有足够的资源。 + a) 根据EM实例信息，找到该EM提供的资源类型，再找到对应的RequestResourceService（有多个子类，每个子类均与一种或多种资源类型对应，有各自的处理逻辑）。 + b) RequestResourceService从多个维度统计剩余的可用资源。 +   i. 根据该EM的总资源，减去已用资源和保护资源后，得出剩余的EM可用资源。 +   ii. 根据该Creator允许使用的资源上限，减去该creator已使用的资源后，得出剩余的Creator可用资源。 +   iii. 根据该用户允许使用的资源上限，减去该用户已使用的资源后，得出剩余的用户可用资源。 +   iv. 根据该用户全局的实例数上限，减去该用户已启动的引擎个数，得出剩余的可用实例数。 + c) 分步骤将剩余可用数量与申请的资源进行比较。 +   i. 按b中所列的顺序，一旦某个步骤的剩余可用数量小于申请的数量时，立刻判定无足够资源，返回NotEnoughResource以及相应的提示信息，不再进行后续步骤的判定。 +   ii. 以上步骤里，如果直到最后剩余可用数量都大于申请的数量，则判定有足够资源，进行下一步锁定资源。 +3. 为成功申请到资源的请求锁定资源。确认资源足够后，为该申请提前锁定资源，并生成唯一标识。 + a) 为了保证并发场景下的正确性，进行锁定操作之前，需要加两个锁（锁机制的具体实现在另外章节中描述）：EM锁和用户锁。 +   i. EM锁。获得该锁以后，将不允许其它针对该EM的资源操作。 +   ii. 用户锁。获得该锁以后，将不允许该用户的其它资源操作。 + b) 在两个锁均成功获得后，将再次重复判断一遍资源是否足够，如果依然足够，则继续进行后续步骤。 + c) 为该资源申请生成一个UUID，并在linkis_user_resource_meta_data表中插入一条用户资源记录（pre_used_resource为申请的资源数量，used_resource为null）。 + d) 在linkis_module_resource_meta_data表中更新对应的EM资源记录字段（locked_resource,left_resource）。 + e) 提交一个定时任务，该任务如果不被取消，则在固定时间后回滚c、d两步的操作，并将UUID作废，以便未被实际使用的已锁定资源不会被无限占据。 + f) 将UUID返回给资源申请方。 + g) 无论以上步骤中发生了什么，都在最后释放a中获得的两个锁。 +4. 接收用户上报的实际使用资源。 + a) 提供resourceInited接口，接受UUID、用户名、EM实例信息、实际使用Resource对象和引擎实例信息作为参数。 + b) 接收到上报信息后，获得EM锁和用户锁。 + c) 根据UUID查询到对应的锁定资源的记录，将pre_used_resource更新为null，将实际使用的资源填写used_resource。 + d) 更新对应的模块资源记录(恢复locked_resource,新增used_resource)。 + e) 异常情况：如果找不到对应的UUID，则认为已经丢失对资源的锁定，返回异常信息。 +5. 接收用户释放资源的请求。 + a) 提供resourceReleased接口，接受UUID、用户名、EM实例作为参数。 + b) 接收到请求后，获得EM锁和用户锁。 + c) 根据UUID查询到对应的用户资源记录，删除该行。 + d) 更新对应的模块资源记录(清理used_resource，恢复left_resource)。 + +## EM锁与用户锁的实现 +通过linkis_resource_lock表来实现锁，利用数据库本身的unique constraint机制保证数据不被抢写。 +1. EM锁：针对全局锁住对某个EM的某个实例的操作。 + a) 获得锁： +   i. 检查是否存在user为null、且application和instance栏位为对应值的记录，若有，则说明该锁已被其它实例获得，轮询等待。 +   ii. 当发现没有对应记录时，插入一条记录，若插入成功，则说明成功获得锁；若插入遇到违反UniqueConstraint错误，则记录轮询等待，直到timeout。 + b) 释放锁： +   i. 删除自己所持有的那行记录。 +2. 用户锁：针对某个用户锁住对某个EM的操作。 + a) 获得锁： +   i. 检查是否存在user，application和instance栏位为对应值的记录，若有，则说明该锁已被其它实例获得，轮询等待。 +   ii. 当发现没有对应记录时，插入一条记录，若插入成功，则说明成功获得锁；若插入失败，则记录轮询等待，直到timeout。 + b) 释放锁： +   i. 删除自己所持有的记录。 + +## RM客户端 +以jar包的形式，提供客户端给资源使用方和资源提供方，包含以下内容： +1. 所有资源类型的Java类（Resource类的子类），以及相应的json序列化方法。 +2. 所有资源分配结果的Java类（ResultResource类的子类），以及相应的json序列化方法。 +3. 封装好的RM接口（资源注册、下线、申请以及可用资源和释放资源的请求）。 +调用客户端的接口后，客户端将生成对应的RPC命令，通过Sender传递给RM的一个微服务进行处理。RM处理完毕后，同样通过RPC将结果返回给客户端。 + +## 多实例状态同步 +  由于RM属于关键的底层服务，为了防止因为某个RM实例出现异常而影响所有服务的资源分配，必须保证同时有多个RM实例处于服务状态，并保证一个请求无论是被哪个实例处理，均能保证结果的一致性。 +  用户在请求RM的服务时，必须通过网关服务的转发来请求，而无法直接请求某一台固定的RM实例。网关服务通过服务注册与发现机制，识别出正常提供服务的RM实例，从而将RPC请求转发给其中一个实例。这就保证了所有请求均会被正常状态的RM实例处理。 +  RM所有的资源记录均存储在同一个数据库中，所有的RM实例均不维护自身的状态。RM在处理请求时，凡是涉及状态变化的，都会在加锁后，从数据库实时获取状态信息，完成处理逻辑后立刻将状态更新回数据库，再释放锁。这就保证了多个RM同时处理请求时，总能基于最新的状态。 + diff --git "a/docs/zh_CN/ch4/Linkis-UJES\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/docs/zh_CN/ch4/Linkis-UJES\350\256\276\350\256\241\346\226\207\346\241\243.md" new file mode 100644 index 00000000000..0f2daa9b0b6 --- /dev/null +++ "b/docs/zh_CN/ch4/Linkis-UJES\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -0,0 +1,115 @@ +### UJES设计文档 +#### 1.文档概述 +##### 1.1项目背景 +UJES(Unified Job Execution Service)，统一作业执行服务是Linkis的核心组件之一。项目以微服务架构的实现方式，为大数据生态圈提供了一种大数据作业提交的全新通用框架方案，同时解决现有市面上同类开源项目的一些使用上的痛点。
+本文档适合于对大数据功能平台，特别是hadoop大数据生态圈，有一定工作经验或有学习兴趣的人士阅读。 +##### 1.2名词解释 +文档会在后文使用到一些项目中的专有名词，在本节中对这些名词进行解释。 + +1) 网关: +UJES的网关，基于Spring Cloud Gateway进行了插件化功能增强，新增了WebSocket一对多能力的网关服务，主要用于转发用户请求到指定的微服务。 + +2) 入口微服务:UJES的入口微服务是用户某一类作业全程的管理者。从作业生成到提交到执行引擎，再到作业信息反馈给用户和作业关闭，入口微服务管理了一个作业的生命周期。 + +3) 引擎管理器:UJES的引擎管理器是处理启动引擎请求的微服务，同时也监控引擎的生命状态。 +4) 执行引擎:UJES的执行引擎是真正执行用户作业的微服务，它由引擎管理器启动，并和提交给它作业的入口进行交互，将作业正确执行，并反馈用户需要的各种信息。 +5) 应用管理器:UJES的应用管理微服务是集群中执行引擎实例信息的维系者，入口微服务想要执行作业，总是需要这些信息来获取一个可以使用的执行引擎。 + +#### 2.整体架构 +正确、安全地连接用户和数据，并为用户提供强大且易用的数据作业提交方式是UJES项目的目标。UJES的定位是连接用户和数据及集群的桥梁。用户只需要将自己的大数据作业提交到UJES，UJES会将作业提交到集群上进行执行，作业在集群上运行产生的日志、状态、进度、结果等都会由UJES实时返回给用户。
+UJES的整体架构如图2.1所示。如图所示，UJES框架位于用户与集群之间，是用户作业的管理角色，它将大数据集群的数据存储、计算等功能进行封装，提供了一种统一的作业提交框架，用户不再需要区分作业是spark还是hive等类型，只需要提交给UJES，就可以正确地让集群为自己服务，节省了用户极大的学习成本。 + +![UJES整体架构图](../images/ch4/UJES整体架构图.png)
+ +图2.1 UJES整体架构图 + +#### 3.逻辑架构 +UJES的逻辑架构是基于现今流行的微服务架构模式进行设计。微服务架构提倡将后台服务划分成一组小的服务，各个微服务之间相互协调、互相配合。微服务和微服务之间采用轻量级的通信机制相互沟通（通常是基于HTTP的Restful API)。这种架构模式具有逻辑清晰、部署简单、扩展性强、技术异构、高可靠等优势。UJES的逻辑架构如图3.1所示。 + +![UJES逻辑架构图](../images/ch4/UJES逻辑架构图.png)
+ +图3.1 UJES逻辑架构图总览 + +##### 3.1 UJES作业执行主流程 +下面将用一个完整的例子来详细阐述介绍UJES项目运行的主流程，并对图中微服务组件的功能进行介绍，更多细节会在主流程之后进行阐述。 +- **1.用户提交作业，网关转发** + + 用户A通过Restful或者Websocket等方式提交自己的大数据作业到UJES的网关微服务，网关根据指定的作业类型，转发用户请求到指定的入口微服务，如用户提交了一段spark-sql代码，网关会将该作业提交到Spark的入口微服务中。由于入口微服务可以是多实例部署的，网关是按照负载均衡的策略进行转发到合适的微服务实例。 +- **2.入口程序进行解析、检查** + + 用户的作业被转发到Spark的入口微服务之后，入口中的解析器会将用户提交的作业解析成一个可用于运行的任务，持久器会将任务持久化到数据库。同时预设置的拦截器也会进行脚本的自定义变量替换，恶意代码检查等操作。如果用户的请求被拦截，那么他的代码将不会被提交到集群进行执行。 +- **3.监听器的设置** + + 任务运行产生的日志、进度、状态、结果等信息产生之后需要进行处理，例如展现给用户或持久化到数据库等，一般这种处理需要采用事件总线和监听器的方式，所以需要为任务设置各类监听器。 +- **4.任务进入调度器** + + 任务生成之后，会进入调度器中等待调度执行。调度器中的核心概念是消费队列，消费队列是以消费组作为标识的，消费组一般是以用户所在的系统和用户两者共同确定，如用户Anna在系统Test中向UJES提交了任务，那么消费组可以标记为Test_Anna。任务进入调度器之后，会根据组的标识，放置到不同的消费队列等待调度。消费队列的线程一般采用单线程的方式实现。 +- **5.应用管理微服务的工作——提供执行引擎** + + 任务一旦被调度，入口微服务就需要通过引擎申请器向应用管理微服务进行申请执行引擎用于执行。 +应用管理微服务会根据用户的消费组信息查看集群中是否有用户可以使用的引擎，如果有可以用于消费组使用的执行引擎，会将该引擎的信息返回给入口微服务，入口微服务会将任务提交到该执行引擎进行执行。 +如果应用管理微服务发现在集群中没有该消费组可以使用的引擎，就会向引擎管理器微服务为该消费组申请一个新的执行引擎。入口微服务的消费线程会一直等待直到应用管理微服务返回引擎启动成功、失败或者超时的信息。 +- **6.引擎管理器微服务——启动并管理引擎** + + 引擎管理器微服务是用于启动并管理执行引擎的微服务。当引擎管理器收到应用管理微服务启动一个新引擎的请求，请求会携带用户的消费组信息，引擎管理器根据消费组信息去向资源管理器申请资源，如果用户仍然有足够的资源，资源管理器就会允许引擎管理器为用户启动一个新的引擎，并广播到应用管理服务微服务。 +- **7.入口微服务提交任务到执行引擎进行执行** + + 步骤7之后，应用管理微服务已经获取到了新启动的引擎的信息，应用管理微服务会将这个引擎的信息返回给入口微服务，入口微服务获取到这个引擎信息之后，就将任务提交到改引擎去执行。 +- **8.入口与引擎的交互** + + 任务提交到执行引擎之后，任务在运行的时候，会产生日志、进度、结果等信息，这些信息会通过RPC的方式进行返回到入口微服务中，返回信息都会携带任务的唯一标识信息，根据任务的标识信息，入口微服务将这些信息正确地进行处理。 +- **9.任务的完成** + + 任务在执行引擎上运行完毕之后，会将运行成功或者失败的状态信息返回给入口微服务，入口微服务中的任务状态翻转之后，消费队列会继续消费在队列中的任务。 +##### 3.2 架构的细节以及优化 +除3.1小节中描述的主流程之外，UJES在集群管理和性能提优方面也有自己的处理流程。 +- **1. 任务分类与多样的消费方式** + + 任务可以按照自身的特点可以分类成以下几类:全新任务、重试任务、重复任务等。全新任务是指用户新提交的任务，重试任务是指在某些特定情况下运行失败需要进行重试的任务，重复任务是指与以前提交任务是一致的任务。 +任务进入调度器的消费队列之后，如果是全新任务，就会进入FIFO的消费器(Consumer)进行消费，如果是重复任务，则会进入ReUse的消费器进行消费，ReUse的消费器所做的事情就会比FIFO简单很多，只需要将原来一样的任务的结果返回给用户就行。 +- **2. 引擎并发数的控制** + + UJES中，一个用户可以启动的引擎是受到控制的，例如一个用户最多可以启动3个Spark引擎。并发数的控制是在入口微服务和资源管理器微服务两者双重保证进行控制的。入口微服务中，每个用户的消费线程至多只有3个处于运行中的任务，这样最多就只会使用三个引擎。资源管理微服务也做了这样的保证，如果一个用户要启动第四个引擎，引擎管理器需要为这个用户去向资源管理微服务请求资源，资源管理微服务会以引擎数量超过限制为由拒绝为该用户提供资源，第四个引擎就会启动失败。 +- **3. 执行引擎的心跳维系与不健康引擎** + + 应用管理微服务在获取到引擎信息之后，需要和引擎进行心跳的维系，以确保引擎的进程是仍然存在的。如果引擎一段时间内没有心跳返回，就会将该引擎加入不健康引擎里面，这样入口请求引擎的时候，这些不健康引擎将不会被使用。 +- **4.引擎的自然消亡与用户主动杀死** + + 引擎的存在是会占用集群资源的，特别是Spark引擎会占用较多的队列资源，所以如果引擎管理器检测到一个执行引擎长时间没有被使用，那么需要将该引擎进行杀死，释放集群的资源，正确杀死引擎之后需要向应用管理器进行广播。用户在使用UJES的时候也会有主动kill引擎的意愿，用户提交杀死引擎的Restful请求到网关，网关转发到引擎管理器，引擎管理器对引擎进行杀死。 +- **5. 多租户隔离** + + 多租户隔离是大数据功能平台的一个重要功能,UJES配合hadoop生态圈组件的权限管理从架构上就天然支持多租户隔离。用户作业是在执行引擎上进行执行，UJES的资源管理微服务在启动一个新的执行引擎的时候，会切换到该用户去执行系统命令，这样执行引擎进程的权限就是该用户的权限，与其他用户启动的引擎是完全隔离的，这样就实现了多租户隔离的功能。 +- **6. 智能诊断** + + 智能诊断是UJES的一个调优模块，大数据作业在执行的时候往往会用到大量数据做计算，同时也会需要集群中大量的资源，一个作业的执行时间会需要较长的时间。用户总是希望能够得到集群的反馈，比如数据是否是倾斜，队列资源是否是足够等。智能诊断就是为这种需求所设定的，诊断模块可以在作业运行时对用户作业的资源和数据进行分析，并把分析的内容实时推送给用户。 +#### 4.接口设计 +##### 4.1 对外接口设计 +UJES对外接口是指与用户使用和与集群的接口。 +- **1 用户接口** + + UJES的使用用户接入UJES的方式一般有Restful、WebSocket两种方式。用户需要将自己的请求按照规定的格式封装成Json，然后通过post的方式提交自己的请求。推荐用户使用采用基于web的方式接入UJES。数据交换的规范会在文后给出。 +- **2 集群接口** + + UJES与集群的交互方式是根据引擎类型决定的。如图2.1所示，UJES的执行引擎是横跨了UJES和集群两个层面。举一个例子，Spark执行引擎是通过Spark提供的Driver API与集群进行交互。用户在使用UJES框架的时候，可以根据自身的需求和特点进行对集群或者是其他服务器资源进行接口接入。 + +##### 4.2 框架接入接口设计 +UJES作为一个框架，框架开发用户可以根据自己的需求进行接入开发。框架的接入一般采用SDK的方式，用户通过maven或gradle等依赖管理引入UJES的SDK之后，需要实现以下几个接口就可以使用UJES框架。 + +1) 入口微服务接入接口 +2) 引擎管理器接入接口 +3) 引擎接入接口 + +具体可以查看UJES的接入文档。 +##### 4.3 内部功能模块接口设计 +UJES内部功能模块之间的交互采用了自研的基于Feign的RPC方式。RPC框架将服务调用方和服务提供方抽象成了Sender和Receiver，服务和服务之间采用Sender通过异步的send方法或者同步的ask方法进行调用。send和ask方法传参是希望Receiver进行处理的类实例。类实例json序列化之后，传到服务提供方，服务提供方的Receiver一般是单例实现，Receiver类有接收异步请求的receive方法和接收同步请求的receiveAndReply方法。Receiver在接收到请求之后，会通过scala的match case语法进行相应的操作。熟悉spark的用户可能比较熟悉这一个处理逻辑，这套API的设计方法也有借鉴Spark的RPC的实现。RPC框架的逻辑如图4.1所示。
+举个例子，微服务A需要调用微服务B的持久化功能，首先微服务A需要获取微服务B在服务注册中心的应用名，通过应用名可以通过RPC框架获取一个Sender，然后将想要持久化的类实例通过send或ask的方式发送到微服务B。微服务B 接收到这个请求之后，通过实例的类信息推断出是一个持久化的请求，根据请求进行后续的操作。如果是同步的请求，操作完成之后需要返回一个消息给调用方。 + +![RPC框架](../images/ch4/RPC框架.png)
+ +图4.1 UJES RPC框架描述 + +#### 5.部署架构 +##### 5.1传统部署方式 +请查看[快速部署文档](../ch1/deploy.md)。 +##### 5.2 容器化部署 +暂无。 diff --git "a/docs/zh_CN/ch5/Hive\345\274\225\346\223\216\344\273\213\347\273\215.md" "b/docs/zh_CN/ch5/Hive\345\274\225\346\223\216\344\273\213\347\273\215.md" new file mode 100644 index 00000000000..e69de29bb2d diff --git "a/docs/zh_CN/ch5/Python\345\274\225\346\223\216\344\273\213\347\273\215.md" "b/docs/zh_CN/ch5/Python\345\274\225\346\223\216\344\273\213\347\273\215.md" new file mode 100644 index 00000000000..e69de29bb2d diff --git "a/docs/zh_CN/ch5/Spark\345\274\225\346\223\216\344\273\213\347\273\215.md" "b/docs/zh_CN/ch5/Spark\345\274\225\346\223\216\344\273\213\347\273\215.md" new file mode 100644 index 00000000000..e69de29bb2d diff --git a/docs/zh_CN/images/ch1/Eureka_homepage.png b/docs/zh_CN/images/ch1/Eureka_homepage.png new file mode 100644 index 00000000000..8564424fd73 Binary files /dev/null and b/docs/zh_CN/images/ch1/Eureka_homepage.png differ diff --git a/docs/zh_CN/images/ch3/dev-sdk/CommonEntranceParser.png b/docs/zh_CN/images/ch3/dev-sdk/CommonEntranceParser.png new file mode 100644 index 00000000000..e4b0d6e2102 Binary files /dev/null and b/docs/zh_CN/images/ch3/dev-sdk/CommonEntranceParser.png differ diff --git "a/docs/zh_CN/images/ch3/\345\212\237\350\203\275\346\250\241\345\235\227\345\233\276.png" "b/docs/zh_CN/images/ch3/\345\212\237\350\203\275\346\250\241\345\235\227\345\233\276.png" new file mode 100644 index 00000000000..15dd01fe365 Binary files /dev/null and "b/docs/zh_CN/images/ch3/\345\212\237\350\203\275\346\250\241\345\235\227\345\233\276.png" differ diff --git "a/docs/zh_CN/images/ch3/\345\216\237\345\244\247\346\225\260\346\215\256\347\224\237\346\200\201\345\233\276.jpg" "b/docs/zh_CN/images/ch3/\345\216\237\345\244\247\346\225\260\346\215\256\347\224\237\346\200\201\345\233\276.jpg" new file mode 100644 index 00000000000..d33ae02365c Binary files /dev/null and "b/docs/zh_CN/images/ch3/\345\216\237\345\244\247\346\225\260\346\215\256\347\224\237\346\200\201\345\233\276.jpg" differ diff --git "a/docs/zh_CN/images/ch3/\350\246\201\350\247\243\345\206\263\347\232\204\351\227\256\351\242\230.jpg" "b/docs/zh_CN/images/ch3/\350\246\201\350\247\243\345\206\263\347\232\204\351\227\256\351\242\230.jpg" new file mode 100644 index 00000000000..7cade3ae5dd Binary files /dev/null and "b/docs/zh_CN/images/ch3/\350\246\201\350\247\243\345\206\263\347\232\204\351\227\256\351\242\230.jpg" differ diff --git "a/docs/zh_CN/images/ch3/\350\247\243\345\206\263\346\226\271\346\241\210.jpg" "b/docs/zh_CN/images/ch3/\350\247\243\345\206\263\346\226\271\346\241\210.jpg" new file mode 100644 index 00000000000..b437314d175 Binary files /dev/null and "b/docs/zh_CN/images/ch3/\350\247\243\345\206\263\346\226\271\346\241\210.jpg" differ diff --git "a/docs/zh_CN/images/ch4/Linkis RPC\346\236\266\346\236\204\346\200\273\345\233\276.png" "b/docs/zh_CN/images/ch4/Linkis RPC\346\236\266\346\236\204\346\200\273\345\233\276.png" new file mode 100644 index 00000000000..98bbd9d3a01 Binary files /dev/null and "b/docs/zh_CN/images/ch4/Linkis RPC\346\236\266\346\236\204\346\200\273\345\233\276.png" differ diff --git "a/docs/zh_CN/images/ch4/Linkis RPC\350\257\246\347\273\206\346\236\266\346\236\204\345\233\276.png" "b/docs/zh_CN/images/ch4/Linkis RPC\350\257\246\347\273\206\346\236\266\346\236\204\345\233\276.png" new file mode 100644 index 00000000000..57316a1be88 Binary files /dev/null and "b/docs/zh_CN/images/ch4/Linkis RPC\350\257\246\347\273\206\346\236\266\346\236\204\345\233\276.png" differ diff --git "a/docs/zh_CN/images/ch4/RM\346\236\266\346\236\204\345\233\276.png" "b/docs/zh_CN/images/ch4/RM\346\236\266\346\236\204\345\233\276.png" new file mode 100644 index 00000000000..4ae1ed1fe04 Binary files /dev/null and "b/docs/zh_CN/images/ch4/RM\346\236\266\346\236\204\345\233\276.png" differ diff --git "a/docs/zh_CN/images/ch4/RPC\346\241\206\346\236\266.png" "b/docs/zh_CN/images/ch4/RPC\346\241\206\346\236\266.png" new file mode 100644 index 00000000000..30ccd9284d1 Binary files /dev/null and "b/docs/zh_CN/images/ch4/RPC\346\241\206\346\236\266.png" differ diff --git "a/docs/zh_CN/images/ch4/UJES/UJES\344\270\232\345\212\241\346\236\266\346\236\204\345\233\276.png" "b/docs/zh_CN/images/ch4/UJES/UJES\344\270\232\345\212\241\346\236\266\346\236\204\345\233\276.png" new file mode 100644 index 00000000000..56be0be8802 Binary files /dev/null and "b/docs/zh_CN/images/ch4/UJES/UJES\344\270\232\345\212\241\346\236\266\346\236\204\345\233\276.png" differ diff --git "a/docs/zh_CN/images/ch4/UJES/UJES\346\212\200\346\234\257\346\236\266\346\236\204\345\233\276.png" "b/docs/zh_CN/images/ch4/UJES/UJES\346\212\200\346\234\257\346\236\266\346\236\204\345\233\276.png" new file mode 100644 index 00000000000..3e8f31927fb Binary files /dev/null and "b/docs/zh_CN/images/ch4/UJES/UJES\346\212\200\346\234\257\346\236\266\346\236\204\345\233\276.png" differ diff --git "a/docs/zh_CN/images/ch4/UJES/UJES\346\265\201\347\250\213\346\227\266\345\272\217\345\233\276.png" "b/docs/zh_CN/images/ch4/UJES/UJES\346\265\201\347\250\213\346\227\266\345\272\217\345\233\276.png" new file mode 100644 index 00000000000..55c0be3b8b6 Binary files /dev/null and "b/docs/zh_CN/images/ch4/UJES/UJES\346\265\201\347\250\213\346\227\266\345\272\217\345\233\276.png" differ diff --git "a/docs/zh_CN/images/ch4/UJES/\345\257\274\345\205\245\346\265\201\347\250\213.png" "b/docs/zh_CN/images/ch4/UJES/\345\257\274\345\205\245\346\265\201\347\250\213.png" new file mode 100644 index 00000000000..a6e097537a1 Binary files /dev/null and "b/docs/zh_CN/images/ch4/UJES/\345\257\274\345\205\245\346\265\201\347\250\213.png" differ diff --git "a/docs/zh_CN/images/ch4/UJES/\345\257\274\345\207\272\346\265\201\347\250\213.png" "b/docs/zh_CN/images/ch4/UJES/\345\257\274\345\207\272\346\265\201\347\250\213.png" new file mode 100644 index 00000000000..b189d390bdc Binary files /dev/null and "b/docs/zh_CN/images/ch4/UJES/\345\257\274\345\207\272\346\265\201\347\250\213.png" differ diff --git "a/docs/zh_CN/images/ch4/UJES\346\225\264\344\275\223\346\236\266\346\236\204\345\233\276.png" "b/docs/zh_CN/images/ch4/UJES\346\225\264\344\275\223\346\236\266\346\236\204\345\233\276.png" new file mode 100644 index 00000000000..e23c19d08be Binary files /dev/null and "b/docs/zh_CN/images/ch4/UJES\346\225\264\344\275\223\346\236\266\346\236\204\345\233\276.png" differ diff --git "a/docs/zh_CN/images/ch4/UJES\351\200\273\350\276\221\346\236\266\346\236\204\345\233\276.png" "b/docs/zh_CN/images/ch4/UJES\351\200\273\350\276\221\346\236\266\346\236\204\345\233\276.png" new file mode 100644 index 00000000000..33c4a93d0db Binary files /dev/null and "b/docs/zh_CN/images/ch4/UJES\351\200\273\350\276\221\346\236\266\346\236\204\345\233\276.png" differ diff --git "a/docs/zh_CN/images/ch4/gateway/Linkis\347\232\204Gateway\346\226\271\346\241\210.png" "b/docs/zh_CN/images/ch4/gateway/Linkis\347\232\204Gateway\346\226\271\346\241\210.png" new file mode 100644 index 00000000000..3f1f2296467 Binary files /dev/null and "b/docs/zh_CN/images/ch4/gateway/Linkis\347\232\204Gateway\346\226\271\346\241\210.png" differ diff --git "a/docs/zh_CN/images/ch4/gateway/Spring Cloud Gateway\347\232\204\345\261\200\351\231\220.png" "b/docs/zh_CN/images/ch4/gateway/Spring Cloud Gateway\347\232\204\345\261\200\351\231\220.png" new file mode 100644 index 00000000000..1744862ea60 Binary files /dev/null and "b/docs/zh_CN/images/ch4/gateway/Spring Cloud Gateway\347\232\204\345\261\200\351\231\220.png" differ diff --git "a/docs/zh_CN/images/ch4/gateway/WebSocket\350\267\257\347\224\261\350\275\254\345\217\221\345\231\250\346\236\266\346\236\204\345\233\276.png" "b/docs/zh_CN/images/ch4/gateway/WebSocket\350\267\257\347\224\261\350\275\254\345\217\221\345\231\250\346\236\266\346\236\204\345\233\276.png" new file mode 100644 index 00000000000..85dd1bc72ae Binary files /dev/null and "b/docs/zh_CN/images/ch4/gateway/WebSocket\350\267\257\347\224\261\350\275\254\345\217\221\345\231\250\346\236\266\346\236\204\345\233\276.png" differ diff --git "a/docs/zh_CN/images/ch4/storage/Dolphin\346\226\207\344\273\266\346\240\274\345\274\217.png" "b/docs/zh_CN/images/ch4/storage/Dolphin\346\226\207\344\273\266\346\240\274\345\274\217.png" new file mode 100644 index 00000000000..f1eceb107b4 Binary files /dev/null and "b/docs/zh_CN/images/ch4/storage/Dolphin\346\226\207\344\273\266\346\240\274\345\274\217.png" differ diff --git "a/docs/zh_CN/images/ch4/storage/Dolphin\346\226\207\344\273\266\350\257\273\345\206\231\346\265\201\347\250\213\345\233\276.png" "b/docs/zh_CN/images/ch4/storage/Dolphin\346\226\207\344\273\266\350\257\273\345\206\231\346\265\201\347\250\213\345\233\276.png" new file mode 100644 index 00000000000..d6d04efed35 Binary files /dev/null and "b/docs/zh_CN/images/ch4/storage/Dolphin\346\226\207\344\273\266\350\257\273\345\206\231\346\265\201\347\250\213\345\233\276.png" differ diff --git "a/docs/zh_CN/images/ch4/storage/Storage\350\277\234\347\250\213\346\250\241\345\274\217\346\236\266\346\236\204.png" "b/docs/zh_CN/images/ch4/storage/Storage\350\277\234\347\250\213\346\250\241\345\274\217\346\236\266\346\236\204.png" new file mode 100644 index 00000000000..02563aaaca9 Binary files /dev/null and "b/docs/zh_CN/images/ch4/storage/Storage\350\277\234\347\250\213\346\250\241\345\274\217\346\236\266\346\236\204.png" differ diff --git "a/docs/zh_CN/images/ch4/storage/\351\200\232\347\224\250\346\226\207\344\273\266\347\263\273\347\273\237\346\226\271\346\241\210.png" "b/docs/zh_CN/images/ch4/storage/\351\200\232\347\224\250\346\226\207\344\273\266\347\263\273\347\273\237\346\226\271\346\241\210.png" new file mode 100644 index 00000000000..01616dce860 Binary files /dev/null and "b/docs/zh_CN/images/ch4/storage/\351\200\232\347\224\250\346\226\207\344\273\266\347\263\273\347\273\237\346\226\271\346\241\210.png" differ diff --git "a/docs/zh_CN/images/ch4/\345\205\250\345\274\202\346\255\245\350\260\203\347\224\250\347\272\277\347\250\213\346\261\240.png" "b/docs/zh_CN/images/ch4/\345\205\250\345\274\202\346\255\245\350\260\203\347\224\250\347\272\277\347\250\213\346\261\240.png" new file mode 100644 index 00000000000..d2bafddc2fc Binary files /dev/null and "b/docs/zh_CN/images/ch4/\345\205\250\345\274\202\346\255\245\350\260\203\347\224\250\347\272\277\347\250\213\346\261\240.png" differ diff --git "a/docs/zh_CN/images/ch4/\345\205\254\345\205\261\346\250\241\345\235\227\346\236\266\346\236\204\350\256\276\350\256\241/\345\210\206\347\273\204\350\260\203\345\272\246\346\266\210\350\264\271\346\236\266\346\236\204.png" "b/docs/zh_CN/images/ch4/\345\205\254\345\205\261\346\250\241\345\235\227\346\236\266\346\236\204\350\256\276\350\256\241/\345\210\206\347\273\204\350\260\203\345\272\246\346\266\210\350\264\271\346\236\266\346\236\204.png" new file mode 100644 index 00000000000..a783ba4576f Binary files /dev/null and "b/docs/zh_CN/images/ch4/\345\205\254\345\205\261\346\250\241\345\235\227\346\236\266\346\236\204\350\256\276\350\256\241/\345\210\206\347\273\204\350\260\203\345\272\246\346\266\210\350\264\271\346\236\266\346\236\204.png" differ diff --git "a/docs/zh_CN/images/ch4/\345\205\254\345\205\261\346\250\241\345\235\227\346\236\266\346\236\204\350\256\276\350\256\241/\350\267\250\346\234\215\345\212\241\345\274\202\346\255\245\345\256\236\346\227\266\346\227\245\345\277\227\346\216\250\351\200\201\346\226\271\346\241\210.png" "b/docs/zh_CN/images/ch4/\345\205\254\345\205\261\346\250\241\345\235\227\346\236\266\346\236\204\350\256\276\350\256\241/\350\267\250\346\234\215\345\212\241\345\274\202\346\255\245\345\256\236\346\227\266\346\227\245\345\277\227\346\216\250\351\200\201\346\226\271\346\241\210.png" new file mode 100644 index 00000000000..44719274181 Binary files /dev/null and "b/docs/zh_CN/images/ch4/\345\205\254\345\205\261\346\250\241\345\235\227\346\236\266\346\236\204\350\256\276\350\256\241/\350\267\250\346\234\215\345\212\241\345\274\202\346\255\245\345\256\236\346\227\266\346\227\245\345\277\227\346\216\250\351\200\201\346\226\271\346\241\210.png" differ diff --git "a/docs/zh_CN/images/ch5/DEBUG\346\210\252\345\233\276.png" "b/docs/zh_CN/images/ch5/DEBUG\346\210\252\345\233\276.png" new file mode 100644 index 00000000000..c306f0cdbe4 Binary files /dev/null and "b/docs/zh_CN/images/ch5/DEBUG\346\210\252\345\233\276.png" differ diff --git a/docs/zh_CN/images/ch6/AbstractEngineCreator.png b/docs/zh_CN/images/ch6/AbstractEngineCreator.png new file mode 100644 index 00000000000..6dd04e50cd2 Binary files /dev/null and b/docs/zh_CN/images/ch6/AbstractEngineCreator.png differ diff --git a/docs/zh_CN/images/ch6/CommonEntranceParser.png b/docs/zh_CN/images/ch6/CommonEntranceParser.png new file mode 100644 index 00000000000..e4b0d6e2102 Binary files /dev/null and b/docs/zh_CN/images/ch6/CommonEntranceParser.png differ diff --git a/docs/zh_CN/images/ch6/EngineExecutor.png b/docs/zh_CN/images/ch6/EngineExecutor.png new file mode 100644 index 00000000000..7bea215a6e5 Binary files /dev/null and b/docs/zh_CN/images/ch6/EngineExecutor.png differ diff --git a/docs/zh_CN/images/ch6/EngineExecutorFactory.png b/docs/zh_CN/images/ch6/EngineExecutorFactory.png new file mode 100644 index 00000000000..e5945fe8151 Binary files /dev/null and b/docs/zh_CN/images/ch6/EngineExecutorFactory.png differ diff --git a/docs/zh_CN/images/ch6/EngineRequesterImpl.png b/docs/zh_CN/images/ch6/EngineRequesterImpl.png new file mode 100644 index 00000000000..6968294ec49 Binary files /dev/null and b/docs/zh_CN/images/ch6/EngineRequesterImpl.png differ diff --git a/docs/zh_CN/images/ch6/EngineResourceFactory.png b/docs/zh_CN/images/ch6/EngineResourceFactory.png new file mode 100644 index 00000000000..6d0e8d9631f Binary files /dev/null and b/docs/zh_CN/images/ch6/EngineResourceFactory.png differ diff --git a/docs/zh_CN/images/ch6/FIFOScheduler.png b/docs/zh_CN/images/ch6/FIFOScheduler.png new file mode 100644 index 00000000000..5973041e095 Binary files /dev/null and b/docs/zh_CN/images/ch6/FIFOScheduler.png differ diff --git a/docs/zh_CN/images/ch6/HiveEngineCreator.png b/docs/zh_CN/images/ch6/HiveEngineCreator.png new file mode 100644 index 00000000000..3121f278477 Binary files /dev/null and b/docs/zh_CN/images/ch6/HiveEngineCreator.png differ diff --git a/docs/zh_CN/images/ch6/HiveEngineExecutor.png b/docs/zh_CN/images/ch6/HiveEngineExecutor.png new file mode 100644 index 00000000000..815d07a646a Binary files /dev/null and b/docs/zh_CN/images/ch6/HiveEngineExecutor.png differ diff --git a/docs/zh_CN/images/ch6/HiveEngineExecutorFactory.png b/docs/zh_CN/images/ch6/HiveEngineExecutorFactory.png new file mode 100644 index 00000000000..8adb613d9ff Binary files /dev/null and b/docs/zh_CN/images/ch6/HiveEngineExecutorFactory.png differ diff --git a/docs/zh_CN/images/ch6/HiveQLProcessBuilder.png b/docs/zh_CN/images/ch6/HiveQLProcessBuilder.png new file mode 100644 index 00000000000..f4539f0aa8b Binary files /dev/null and b/docs/zh_CN/images/ch6/HiveQLProcessBuilder.png differ diff --git a/docs/zh_CN/images/ch6/codeParser_bean.png b/docs/zh_CN/images/ch6/codeParser_bean.png new file mode 100644 index 00000000000..1965ec707c2 Binary files /dev/null and b/docs/zh_CN/images/ch6/codeParser_bean.png differ diff --git a/docs/zh_CN/images/ch6/engineCreator_bean.png b/docs/zh_CN/images/ch6/engineCreator_bean.png new file mode 100644 index 00000000000..3121f278477 Binary files /dev/null and b/docs/zh_CN/images/ch6/engineCreator_bean.png differ diff --git a/docs/zh_CN/images/ch6/engineHooks_bean.png b/docs/zh_CN/images/ch6/engineHooks_bean.png new file mode 100644 index 00000000000..dab9efc06cf Binary files /dev/null and b/docs/zh_CN/images/ch6/engineHooks_bean.png differ diff --git a/docs/zh_CN/images/ch6/engineParser_bean.png b/docs/zh_CN/images/ch6/engineParser_bean.png new file mode 100644 index 00000000000..372b231dedc Binary files /dev/null and b/docs/zh_CN/images/ch6/engineParser_bean.png differ diff --git a/docs/zh_CN/images/ch6/group.png b/docs/zh_CN/images/ch6/group.png new file mode 100644 index 00000000000..a40d40d80b7 Binary files /dev/null and b/docs/zh_CN/images/ch6/group.png differ diff --git a/docs/zh_CN/images/ch6/hive_beans.png b/docs/zh_CN/images/ch6/hive_beans.png new file mode 100644 index 00000000000..77ce2aacbbd Binary files /dev/null and b/docs/zh_CN/images/ch6/hive_beans.png differ diff --git a/docs/zh_CN/images/ch6/hive_engineResourceFactory_bean.png b/docs/zh_CN/images/ch6/hive_engineResourceFactory_bean.png new file mode 100644 index 00000000000..a966a79d5df Binary files /dev/null and b/docs/zh_CN/images/ch6/hive_engineResourceFactory_bean.png differ diff --git a/docs/zh_CN/images/ch6/hive_run1.png b/docs/zh_CN/images/ch6/hive_run1.png new file mode 100644 index 00000000000..8b3c3d5ee39 Binary files /dev/null and b/docs/zh_CN/images/ch6/hive_run1.png differ diff --git a/docs/zh_CN/images/ch6/hive_run2.png b/docs/zh_CN/images/ch6/hive_run2.png new file mode 100644 index 00000000000..4f99ad9b68a Binary files /dev/null and b/docs/zh_CN/images/ch6/hive_run2.png differ diff --git a/docs/zh_CN/images/ch6/hooks.png b/docs/zh_CN/images/ch6/hooks.png new file mode 100644 index 00000000000..8f47261b26b Binary files /dev/null and b/docs/zh_CN/images/ch6/hooks.png differ diff --git a/docs/zh_CN/images/ch6/python_run1.png b/docs/zh_CN/images/ch6/python_run1.png new file mode 100644 index 00000000000..1f21ea047c3 Binary files /dev/null and b/docs/zh_CN/images/ch6/python_run1.png differ diff --git a/docs/zh_CN/images/ch6/python_run2.png b/docs/zh_CN/images/ch6/python_run2.png new file mode 100644 index 00000000000..50b241c9fd2 Binary files /dev/null and b/docs/zh_CN/images/ch6/python_run2.png differ diff --git a/docs/zh_CN/images/ch6/resources_bean.png b/docs/zh_CN/images/ch6/resources_bean.png new file mode 100644 index 00000000000..ed7e02fbdd2 Binary files /dev/null and b/docs/zh_CN/images/ch6/resources_bean.png differ diff --git a/docs/zh_CN/images/ch6/spark_conf.png b/docs/zh_CN/images/ch6/spark_conf.png new file mode 100644 index 00000000000..17069d9089c Binary files /dev/null and b/docs/zh_CN/images/ch6/spark_conf.png differ diff --git a/docs/zh_CN/images/ch6/spark_run1.png b/docs/zh_CN/images/ch6/spark_run1.png new file mode 100644 index 00000000000..b6c5d874d42 Binary files /dev/null and b/docs/zh_CN/images/ch6/spark_run1.png differ diff --git a/docs/zh_CN/images/ch6/spark_run2.png b/docs/zh_CN/images/ch6/spark_run2.png new file mode 100644 index 00000000000..4e543ec00cb Binary files /dev/null and b/docs/zh_CN/images/ch6/spark_run2.png differ diff --git a/docs/zh_CN/images/ch6/spark_run3.png b/docs/zh_CN/images/ch6/spark_run3.png new file mode 100644 index 00000000000..02d9cfc72a0 Binary files /dev/null and b/docs/zh_CN/images/ch6/spark_run3.png differ diff --git a/docs/zh_CN/images/introduction/comparison_table.png b/docs/zh_CN/images/introduction/comparison_table.png new file mode 100644 index 00000000000..088faf934b7 Binary files /dev/null and b/docs/zh_CN/images/introduction/comparison_table.png differ diff --git a/docs/zh_CN/images/introduction/introduction03.jpg b/docs/zh_CN/images/introduction/introduction03.jpg new file mode 100644 index 00000000000..b74ab42601f Binary files /dev/null and b/docs/zh_CN/images/introduction/introduction03.jpg differ diff --git a/docs/zh_CN/images/introduction/introduction04.jpg b/docs/zh_CN/images/introduction/introduction04.jpg new file mode 100644 index 00000000000..0a05bd2e351 Binary files /dev/null and b/docs/zh_CN/images/introduction/introduction04.jpg differ diff --git a/docs/zh_CN/images/introduction/introduction05.png b/docs/zh_CN/images/introduction/introduction05.png new file mode 100644 index 00000000000..01ec041d7f1 Binary files /dev/null and b/docs/zh_CN/images/introduction/introduction05.png differ diff --git a/docs/zh_CN/images/introduction/introduction_new.png b/docs/zh_CN/images/introduction/introduction_new.png new file mode 100644 index 00000000000..06c921bd841 Binary files /dev/null and b/docs/zh_CN/images/introduction/introduction_new.png differ diff --git a/eurekaServer/bin/start-eureka.sh b/eurekaServer/bin/start-eureka.sh index 75bbe714736..17675f78ccb 100644 --- a/eurekaServer/bin/start-eureka.sh +++ b/eurekaServer/bin/start-eureka.sh @@ -15,7 +15,7 @@ if [[ -f "${EUREKA_SERVER_PID}" ]]; then fi export EUREKA_SERVER_LOG_PATH=$HOME/logs -export EUREKA_SERVER_HEAP_SIZE="2G" +export EUREKA_SERVER_HEAP_SIZE="1G" export EUREKA_SERVER_CLASS=${EUREKA_SERVER_CLASS:-com.webank.wedatasphere.linkis.eureka.SpringCloudEurekaApplication} profiles='eureka' diff --git a/eurekaServer/pom.xml b/eurekaServer/pom.xml index 18024b88351..1ee9feb96bf 100644 --- a/eurekaServer/pom.xml +++ b/eurekaServer/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-eureka-server diff --git a/eurekaServer/src/main/resources/log4j.properties b/eurekaServer/src/main/resources/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/eurekaServer/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/extensions/spark-excel/pom.xml b/extensions/spark-excel/pom.xml new file mode 100644 index 00000000000..05876ce9a62 --- /dev/null +++ b/extensions/spark-excel/pom.xml @@ -0,0 +1,180 @@ + + + 4.0.0 + + com.webank.weDataSphere + spark-excel_2.11 + 0.8.0 + + + UTF-8 + UTF-8 + 1.7 + 3.3.3 + 2.11.7 + 2.11 + 2.1.0 + 2.7.2 + 0.8.2.0 + 1.7.12 + 1.2.2 + 2.7.0 + 1.1.2 + 1.2.17 + 3.2.11 + 12.0.1 + provided + provided + + + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + org.apache.poi + poi-ooxml + 3.17 + + + com.norbitltd + spoiwo_${scala.compat.version} + 1.2.0 + + + com.monitorjbl + xlsx-streamer + 1.2.1 + + + org.apache.poi + ooxml-schemas + + + + + com.fasterxml.jackson.module + jackson-module-scala_${scala.compat.version} + 2.8.8 + ${spark-scope} + + + org.scalatest + scalatest_${scala.compat.version} + 3.0.1 + ${spark-scope} + + + org.scalacheck + scalacheck_${scala.compat.version} + 1.13.4 + ${spark-scope} + + + com.github.alexarchambault + scalacheck-shapeless_1.13_${scala.compat.version} + 1.1.6 + ${spark-scope} + + + com.holdenkarau + spark-testing-base_${scala.compat.version} + 2.1.0_0.9.0 + ${spark-scope} + + + org.scalamock + scalamock-scalatest-support_${scala.compat.version} + 3.5.0 + ${spark-scope} + + + + org.apache.spark + spark-core_${scala.compat.version} + ${spark.version} + ${spark-scope} + + + + org.apache.spark + spark-sql_${scala.compat.version} + ${spark.version} + ${spark-scope} + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop.version} + ${spark-scope} + + + servlet-api + javax.servlet + + + com.google.guava + guava + + + + + + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + ${java.version} + ${java.version} + UTF-8 + + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + incremental + + -target:jvm-${java.version} + + + -source + ${java.version} + -target + ${java.version} + + + + + process-resources + + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + + + \ No newline at end of file diff --git a/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/DefaultSource.scala b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/DefaultSource.scala new file mode 100644 index 00000000000..884b9b17e66 --- /dev/null +++ b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/DefaultSource.scala @@ -0,0 +1,97 @@ +package com.webank.wedatasphere.spark.excel + +import org.apache.hadoop.fs.Path +import org.apache.spark.sql.sources._ +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode} + +class DefaultSource extends RelationProvider with SchemaRelationProvider with CreatableRelationProvider { + + /** + * Creates a new relation for retrieving data from an Excel file + */ + override def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): ExcelRelation = + createRelation(sqlContext, parameters, null) + + /** + * Creates a new relation for retrieving data from an Excel file + */ + override def createRelation( + sqlContext: SQLContext, + parameters: Map[String, String], + schema: StructType + ): ExcelRelation = { + ExcelRelation( + location = checkParameter(parameters, "path"), + sheetName = parameters.get("sheetName"), + useHeader = checkParameter(parameters, "useHeader").toBoolean, + treatEmptyValuesAsNulls = parameters.get("treatEmptyValuesAsNulls").fold(true)(_.toBoolean), + userSchema = Option(schema), + inferSheetSchema = parameters.get("inferSchema").fold(false)(_.toBoolean), + addColorColumns = parameters.get("addColorColumns").fold(false)(_.toBoolean), + startColumn = parameters.get("startColumn").fold(0)(_.toInt), + endColumn = parameters.get("endColumn").fold(Int.MaxValue)(_.toInt), + timestampFormat = parameters.get("timestampFormat"), + maxRowsInMemory = parameters.get("maxRowsInMemory").map(_.toInt), + excerptSize = parameters.get("excerptSize").fold(10)(_.toInt), + parameters = parameters, + dateFormat = parameters.get("dateFormats").getOrElse("yyyy-MM-dd").split(";").toList + )(sqlContext) + } + + override def createRelation( + sqlContext: SQLContext, + mode: SaveMode, + parameters: Map[String, String], + data: DataFrame + ): BaseRelation = { + val path = checkParameter(parameters, "path") + val sheetName = parameters.getOrElse("sheetName", "Sheet1") + val useHeader = checkParameter(parameters, "useHeader").toBoolean + val dateFormat = parameters.getOrElse("dateFormat", ExcelFileSaver.DEFAULT_DATE_FORMAT) + val timestampFormat = parameters.getOrElse("timestampFormat", ExcelFileSaver.DEFAULT_TIMESTAMP_FORMAT) + val filesystemPath = new Path(path) + val fs = filesystemPath.getFileSystem(sqlContext.sparkContext.hadoopConfiguration) + fs.setWriteChecksum(false) + val doSave = if (fs.exists(filesystemPath)) { + mode match { + case SaveMode.Append => + sys.error(s"Append mode is not supported by ${this.getClass.getCanonicalName}") + case SaveMode.Overwrite => + fs.delete(filesystemPath, true) + true + case SaveMode.ErrorIfExists => + sys.error(s"path $path already exists.") + case SaveMode.Ignore => false + } + } else { + true + } + if (doSave) { + // Only save data when the save mode is not ignore. + (new ExcelFileSaver(fs)).save( + filesystemPath, + data, + sheetName = sheetName, + useHeader = useHeader, + dateFormat = dateFormat, + timestampFormat = timestampFormat + ) + } + + createRelation(sqlContext, parameters, data.schema) + } + + // Forces a Parameter to exist, otherwise an exception is thrown. + private def checkParameter(map: Map[String, String], param: String): String = { + if (!map.contains(param)) { + throw new IllegalArgumentException(s"Parameter ${'"'}$param${'"'} is missing in options.") + } else { + map.apply(param) + } + } + + // Gets the Parameter if it exists, otherwise returns the default argument + private def parameterOrDefault(map: Map[String, String], param: String, default: String) = + map.getOrElse(param, default) +} diff --git a/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/DefaultSource15.scala b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/DefaultSource15.scala new file mode 100644 index 00000000000..27d04ef7095 --- /dev/null +++ b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/DefaultSource15.scala @@ -0,0 +1,7 @@ +package com.webank.wedatasphere.spark.excel + +import org.apache.spark.sql.sources.DataSourceRegister + +class DefaultSource15 extends DefaultSource with DataSourceRegister { + override def shortName(): String = "excel" +} diff --git a/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/ExcelFileSaver.scala b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/ExcelFileSaver.scala new file mode 100644 index 00000000000..5e0ddd14a94 --- /dev/null +++ b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/ExcelFileSaver.scala @@ -0,0 +1,146 @@ +package com.webank.wedatasphere.spark.excel + +import java.io.{BufferedOutputStream, OutputStream} +import java.text.SimpleDateFormat +import java.util.Date + +import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.poi.ss.usermodel.IndexedColors +import org.apache.poi.xssf.streaming.SXSSFWorkbook +import org.apache.poi.ss.usermodel._ +import org.apache.spark.sql.DataFrame + +import scala.language.postfixOps + +object ExcelFileSaver { + final val DEFAULT_SHEET_NAME = "Sheet1" + final val DEFAULT_DATE_FORMAT = "yyyy-MM-dd" + final val DEFAULT_TIMESTAMP_FORMAT = "yyyy-mm-dd hh:mm:ss.000" +} + +class ExcelFileSaver(fs: FileSystem) { + + import ExcelFileSaver._ + + def save( + location: Path, + dataFrame: DataFrame, + sheetName: String = DEFAULT_SHEET_NAME, + useHeader: Boolean = true, + dateFormat: String = DEFAULT_DATE_FORMAT, + timestampFormat: String = DEFAULT_TIMESTAMP_FORMAT + ): Unit = { + fs.setVerifyChecksum(false) + val headerRow = dataFrame.schema.map(_.name) + val dataRows = dataFrame.toLocalIterator() + val excelWriter = new ExcelWriter(sheetName, dateFormat, timestampFormat) + if (useHeader) excelWriter.writeHead(headerRow) + while (dataRows.hasNext) { + val line = dataRows.next().toSeq + excelWriter.writerRow(line) + } + excelWriter.close(new BufferedOutputStream(fs.create(location))) + } + + def autoClose[A <: AutoCloseable, B](closeable: A)(fun: (A) => B): B = { + try { + fun(closeable) + } finally { + closeable.close() + } + } + + +} + +class ExcelWriter(sheetName: String, dateFormat: String, timestampFormat: String) { + + + val wb = new SXSSFWorkbook(100) + val sheet = wb.createSheet(sheetName) + val df = new SimpleDateFormat(dateFormat) + val tf = new SimpleDateFormat(timestampFormat) + val createHelper = wb.getCreationHelper + val sdf = wb.createCellStyle() + sdf.setDataFormat(createHelper.createDataFormat().getFormat(dateFormat)) + val stf = wb.createCellStyle() + stf.setDataFormat(createHelper.createDataFormat().getFormat(timestampFormat)) + + var rowNum = 0 + var columnsLen = 0 + + def writeHead(headRow: Seq[String]): Unit = { + columnsLen = headRow.length + //设置header的格式 + val headerFont = wb.createFont() + headerFont.setBold(true) + headerFont.setFontHeightInPoints(14) + headerFont.setColor(IndexedColors.RED.getIndex()) + val headerCellStyle = wb.createCellStyle() + headerCellStyle.setFont(headerFont) + val row = sheet.createRow(rowNum) + for (i <- headRow.indices) { + createCell(row, i, headRow(i), headerCellStyle) + } + rowNum = rowNum + 1 + } + + def writerRow(line: Seq[Any]): Unit = { + val row = sheet.createRow(rowNum) + for (i <- line.indices) { + createCell(row, i, line(i)) + } + rowNum = rowNum + 1 + } + + def createCell(row: Row, col: Int, value: String, cellStyle: CellStyle): Unit = { + val cell = row.createCell(col) + cell.setCellValue(value) + cell.setCellStyle(cellStyle) + } + + def setDateValue(cell:Cell, date: Date): Unit ={ + cell.setCellStyle(sdf) + cell.setCellValue(date) + } + def setTimestampValue(cell:Cell, date: Date): Unit ={ + cell.setCellStyle(stf) + cell.setCellValue(date) + } + + def createCell(row: Row, col: Int, value: Any): Unit = { + val cell = row.createCell(col) + value match { + case t: java.sql.Timestamp => setTimestampValue(cell, new Date(t.getTime)) + case d: java.sql.Date => setDateValue(cell, new Date(d.getTime)) + case s: String => cell.setCellValue(s) + case f: Float => cell.setCellValue(f) + case d: Double => cell.setCellValue(d) + case b: Boolean => cell.setCellValue(b) + case b: Byte => cell.setCellValue(b) + case s: Short => cell.setCellValue(s) + case i: Int => cell.setCellValue(i) + case l: Long => cell.setCellValue(l) + case b: BigDecimal => cell.setCellValue(b.doubleValue()) + case b: java.math.BigDecimal => cell.setCellValue(b.doubleValue()) + case null => cell.setCellValue("NULL") + case _ => cell.setCellValue(value.toString) + } + } + + def close(outputStream: OutputStream): Unit = { + try { + sheet.trackAllColumnsForAutoSizing() + for (i <- 0 until columnsLen) { + sheet.autoSizeColumn(i) + } + wb.write(outputStream); + } catch { + case e: Throwable => + throw e + } finally { + outputStream.close() + wb.close() + } + } +} diff --git a/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/ExcelRelation.scala b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/ExcelRelation.scala new file mode 100644 index 00000000000..8840d75ba02 --- /dev/null +++ b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/ExcelRelation.scala @@ -0,0 +1,364 @@ +package com.webank.wedatasphere.spark.excel + +import java.math.BigDecimal +import java.sql.Timestamp +import java.text.SimpleDateFormat +import java.util.Locale + +import com.monitorjbl.xlsx.StreamingReader +import org.apache.commons.lang.StringUtils +import org.apache.hadoop.fs.{FileSystem, Path} +import org.apache.poi.ss.usermodel.{Cell, CellType, DataFormatter, DateUtil, Sheet, Workbook, WorkbookFactory, Row => SheetRow} +import org.apache.spark.Logging +import org.apache.spark.rdd.RDD +import org.apache.spark.sql._ +import org.apache.spark.sql.catalyst.util.DateTimeUtils +import org.apache.spark.sql.sources._ +import org.apache.spark.sql.types._ + +import scala.collection.JavaConverters._ +import scala.util.{Failure, Success, Try} + +case class ExcelRelation( + location: String, + sheetName: Option[String], + useHeader: Boolean, + treatEmptyValuesAsNulls: Boolean, + inferSheetSchema: Boolean, + addColorColumns: Boolean = true, + userSchema: Option[StructType] = None, + startColumn: Int = 0, + endColumn: Int = Int.MaxValue, + timestampFormat: Option[String] = None, + maxRowsInMemory: Option[Int] = None, + excerptSize: Int = 10, + parameters: Map[String, String] = null,dateFormat: List[String] +)(@transient val sqlContext: SQLContext) + extends BaseRelation + with TableScan + with PrunedScan with Logging{ + + private val path = new Path(location) + + def extractCells(row: org.apache.poi.ss.usermodel.Row): Vector[Option[Cell]] = + row.eachCellIterator(startColumn, endColumn).to[Vector] + + private def openWorkbook(): Workbook = { + val inputStream = FileSystem.get(path.toUri, sqlContext.sparkContext.hadoopConfiguration).open(path) + maxRowsInMemory + .map { maxRowsInMem => + StreamingReader + .builder() + .rowCacheSize(maxRowsInMem) + .bufferSize(maxRowsInMem * 1024) + .open(inputStream) + } + .getOrElse(WorkbookFactory.create(inputStream)) + } + + /** + * 抽取10row 用于自动推断 + * @return + */ + private def getExcerpt(): (SheetRow, List[SheetRow]) = { + val workbook = openWorkbook() + val sheet = findSheet(workbook, sheetName) + val sheetIterator = sheet.iterator.asScala + var currentRow: org.apache.poi.ss.usermodel.Row = null + while (sheetIterator.hasNext && currentRow == null) { + currentRow = sheetIterator.next + } + if (currentRow == null) { + workbook.close() + throw new RuntimeException(s"Sheet $sheetName in $path doesn't seem to contain any data") + } + val firstRow = currentRow + var counter = 0 + val excerpt = new Array[org.apache.poi.ss.usermodel.Row](excerptSize) + while (sheetIterator.hasNext && counter < excerptSize) { + excerpt(counter) = sheetIterator.next + counter += 1 + } + workbook.close() + (firstRow, excerpt.take(counter).to[List]) + } + + /** + * 删掉已经自动推断的行 + * @param wb + * @param excerptSize + * @return + */ + private def restIterator(wb: Workbook, excerptSize: Int): Iterator[org.apache.poi.ss.usermodel.Row] = { + val sheet = findSheet(wb, sheetName) + val i = sheet.iterator.asScala + i.drop(excerptSize + 1) + i + } + + /** + * 返回推断后的全部数据 + * @param workbook + * @param firstRowWithData + * @param excerpt + * @return + */ + private def dataIterator(workbook: Workbook, firstRowWithData: SheetRow, excerpt: List[SheetRow]) = { + val init = if (useHeader) excerpt else firstRowWithData :: excerpt + init.iterator ++ restIterator(workbook, excerpt.size) + } + + /** + * 遍历所有sheet + * @param wb + * @return + */ + private def dataAllIterator(wb: Workbook): Iterator[org.apache.poi.ss.usermodel.Row] = { + val names = sheetName.getOrElse("sheet1") + if("sheet1".equals(names) || ! names.contains(',')){ + val sheet = findSheet(wb, sheetName) + val i = sheet.iterator.asScala + if (useHeader) i.drop(1) + i + } else { + val sheetNames = names.split(',') + var resIterator = Iterator[org.apache.poi.ss.usermodel.Row]() + for(name <- sheetNames){ + val sheet = findSheet(wb, Some(name)) + val i = sheet.iterator.asScala + if (useHeader) i.drop(1) + resIterator = resIterator ++ i + } + resIterator + } + } + + override val schema: StructType = inferSchema + + /** + * 数据格式转换 + */ + val dataFormatter = new DataFormatter() + + //val dateFormat: SimpleDateFormat = new SimpleDateFormat(parameters.getOrElse( "dateFormat", "yyyy-MM-dd"), Locale.US) + + val nullValue =parameters.getOrElse( "nullValue", " ") + val nanValue = parameters.getOrElse( "nanValue", "null") + val quote = parameters.getOrElse( "quote", "\"") + val escape = parameters.getOrElse( "escape", "\\") + val escapeQuotes = parameters.getOrElse("escapeQuotes", false) + + val timestampParser = if (timestampFormat.isDefined) { + Some(new SimpleDateFormat(timestampFormat.get)) + } else { + None + } + + + /** + * 获得对应的sheet + * @param workBook + * @param sheetName + * @return + */ + private def findSheet(workBook: Workbook, sheetName: Option[String]): Sheet = { + sheetName + .map { sn => + Option(workBook.getSheet(sn)).getOrElse(throw new IllegalArgumentException(s"Unknown sheet $sn")) + } + .getOrElse(workBook.sheetIterator.next) + } + + /** + * build Row + * @return + */ + override def buildScan: RDD[Row] = buildScan(schema.map(_.name).toArray) + + override def buildScan(requiredColumns: Array[String]): RDD[Row] = { + val lookups = requiredColumns + .map { c => + val columnNameRegex = s"(.*?)(_color)?".r + val columnNameRegex(columnName, isColor) = c + val columnIndex = schema.indexWhere(_.name == columnName) + + val cellExtractor: Cell => Any = if (isColor == null) { + castTo(_, schema(columnIndex).dataType,new SimpleDateFormat(dateFormat(columnIndex),Locale.US),null) + } else { + _.getCellStyle.getFillForegroundColorColor match { + case null => "" + case c: org.apache.poi.xssf.usermodel.XSSFColor => c.getARGBHex + case c => throw new RuntimeException(s"Unknown color type $c: ${c.getClass}") + } + } + { row: SheetRow => + val cell = row.getCell(columnIndex + startColumn) + if (cell == null) { + null + } else { + cellExtractor(cell) + } + } + } + .to[Vector] + val workbook = openWorkbook() + val rows = dataAllIterator(workbook).map(row => lookups.map(l => l(row))) + val result = rows.to[Vector] + val rdd = sqlContext.sparkContext.parallelize(result.map(Row.fromSeq)) + workbook.close() + rdd + } + + private def stringToDouble(value: String): Double = { + Try(value.toDouble) match { + case Success(d) => d + case Failure(_) => Double.NaN + } + } + + private def castTo(cell: Cell, castType: DataType,dateFormat: SimpleDateFormat,timestampFormat:SimpleDateFormat): Any = { + val cellType = cell.getCellTypeEnum + if (cellType == CellType.BLANK) { + return null + } + + lazy val dataFormatter = new DataFormatter() + lazy val stringValue = + cell.getCellTypeEnum match { + case CellType.FORMULA => + cell.getCachedFormulaResultTypeEnum match { + case CellType.STRING => cell.getRichStringCellValue.getString + case CellType.NUMERIC => cell.getNumericCellValue.toString + case _ => dataFormatter.formatCellValue(cell) + } + case CellType.ERROR =>logInfo(s"解析excel遇到错误公式数据${cell.getStringCellValue},置为空值");"" + case _ => dataFormatter.formatCellValue(cell) + } + lazy val numericValue = + cell.getCellTypeEnum match { + case CellType.NUMERIC => cell.getNumericCellValue + case CellType.STRING => stringToDouble(cell.getStringCellValue) + case CellType.FORMULA => + cell.getCachedFormulaResultTypeEnum match { + case CellType.NUMERIC => cell.getNumericCellValue + case CellType.STRING => stringToDouble(cell.getRichStringCellValue.getString) + } + } + lazy val bigDecimal = new BigDecimal(numericValue) + castType match { + case _: ByteType => numericValue.toByte + case _: ShortType => numericValue.toShort + case _: IntegerType => numericValue.toInt + case _: LongType => numericValue.toLong + case _: FloatType => numericValue.toFloat + case _: DoubleType => numericValue + case _: BooleanType => cell.getBooleanCellValue + case _: DecimalType => if (cellType == CellType.STRING && (cell.getStringCellValue == "" || "null".equalsIgnoreCase(cell.getStringCellValue))) null else bigDecimal + case _: TimestampType => + cellType match { + case CellType.NUMERIC => new Timestamp(DateUtil.getJavaDate(numericValue).getTime) + case _ => parseTimestamp(stringValue) + } + case _: DateType => + if("null".equalsIgnoreCase(cell.getStringCellValue)||StringUtils.isEmpty(cell.getStringCellValue)){ + null + }else{ + new java.sql.Date(Try(dateFormat.parse(cell.getStringCellValue).getTime).getOrElse(DateUtil.getJavaDate(numericValue).getTime))} + + case _: StringType => stringValue.replaceAll("\n|\t", " ") + case t => throw new RuntimeException(s"Unsupported cast from $cell to $t") + } + } + + private def parseTimestamp(stringValue: String): Timestamp = { + timestampParser match { + case Some(parser) => new Timestamp(parser.parse(stringValue).getTime) + case None => Timestamp.valueOf(stringValue) + } + } + + private def getSparkType(cell: Option[Cell]): DataType = { + cell match { + case Some(c) => + c.getCellTypeEnum match { + case CellType.FORMULA => + c.getCachedFormulaResultTypeEnum match { + case CellType.STRING => StringType + case CellType.NUMERIC => DoubleType + case _ => NullType + } + case CellType.STRING if c.getStringCellValue == "" || "null".equalsIgnoreCase(c.getStringCellValue) => NullType + case CellType.STRING => StringType + case CellType.BOOLEAN => BooleanType + case CellType.NUMERIC => if (DateUtil.isCellDateFormatted(c)) TimestampType else DoubleType + case CellType.BLANK => NullType + } + case None => NullType + } + } + + private def parallelize[T : scala.reflect.ClassTag](seq: Seq[T]): RDD[T] = sqlContext.sparkContext.parallelize(seq) + + /** + * Generates a header from the given row which is null-safe and duplicate-safe. + */ + protected def makeSafeHeader(row: Array[String]): Array[String] = { + if (useHeader) { + val duplicates = { + val headerNames = row + .filter(_ != null) + headerNames.diff(headerNames.distinct).distinct + } + + row.zipWithIndex.map { + case (value, index) => + if (value == null || value.isEmpty) { + // When there are empty strings or the, put the index as the suffix. + s"_c$index" + } else if (duplicates.contains(value)) { + // When there are duplicates, put the index as the suffix. + s"$value$index" + } else { + value + } + } + } else { + row.zipWithIndex.map { + case (_, index) => + // Uses default column names, "_c#" where # is its position of fields + // when header option is disabled. + s"_c$index" + } + } + } + + private def inferSchema(): StructType = this.userSchema.getOrElse { + val (firstRowWithData, excerpt) = getExcerpt() + + val rawHeader = extractCells(firstRowWithData).map { + case Some(value) => value.getStringCellValue + case _ => "" + }.toArray + + val header = makeSafeHeader(rawHeader) + + val baseSchema = if (this.inferSheetSchema) { + val stringsAndCellTypes = excerpt + .map(r => extractCells(r).map(getSparkType)) + InferSchema(parallelize(stringsAndCellTypes), header.toArray) + } else { + // By default fields are assumed to be StringType + val schemaFields = header.map { fieldName => + StructField(fieldName.toString, StringType, nullable = true) + } + StructType(schemaFields) + } + if (addColorColumns) { + header.foldLeft(baseSchema) { (schema, header) => + schema.add(s"${header}_color", StringType, nullable = true) + } + } else { + baseSchema + } + } +} diff --git a/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/InferSchema.scala b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/InferSchema.scala new file mode 100644 index 00000000000..cfc2448afc9 --- /dev/null +++ b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/InferSchema.scala @@ -0,0 +1,94 @@ +package com.webank.wedatasphere.spark.excel + +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.types._ + + +private[excel] object InferSchema { + + type CellType = Int + + /** + * Similar to the JSON schema inference. + * [[org.apache.spark.sql.execution.datasources.json.InferSchema]] + * 1. Infer type of each row + * 2. Merge row types to find common type + * 3. Replace any null types with string type + */ + def apply(rowsRDD: RDD[Seq[DataType]], header: Array[String]): StructType = { + val startType: Array[DataType] = Array.fill[DataType](header.length)(NullType) + val rootTypes: Array[DataType] = rowsRDD.aggregate(startType)(inferRowType _, mergeRowTypes) + + val structFields = header.zip(rootTypes).map { + case (thisHeader, rootType) => + val dType = rootType match { + case z: NullType => StringType + case other => other + } + StructField(thisHeader, dType, nullable = true) + } + StructType(structFields) + } + + private def inferRowType(rowSoFar: Array[DataType], next: Seq[DataType]): Array[DataType] = { + var i = 0 + while (i < math.min(rowSoFar.length, next.size)) { // May have columns on right missing. + rowSoFar(i) = inferField(rowSoFar(i), next(i)) + i += 1 + } + rowSoFar + } + + private[excel] def mergeRowTypes(first: Array[DataType], second: Array[DataType]): Array[DataType] = { + first.zipAll(second, NullType, NullType).map { + case ((a, b)) => + findTightestCommonType(a, b).getOrElse(NullType) + } + } + + /** + * Infer type of string field. Given known type Double, and a string "1", there is no + * point checking if it is an Int, as the final type must be Double or higher. + */ + private[excel] def inferField(typeSoFar: DataType, field: DataType): DataType = { + // Defining a function to return the StringType constant is necessary in order to work around + // a Scala compiler issue which leads to runtime incompatibilities with certain Spark versions; + // see issue #128 for more details. + def stringType(): DataType = { + StringType + } + + if (field == NullType) { + typeSoFar + } else { + (typeSoFar, field) match { + case (NullType, ct) => ct + case (DoubleType, DoubleType) => DoubleType + case (BooleanType, BooleanType) => BooleanType + case (TimestampType, TimestampType) => TimestampType + case (StringType, _) => stringType() + case (_, _) => stringType() + } + } + } + + + private val numericPrecedence: IndexedSeq[DataType] = + IndexedSeq[DataType](ByteType, ShortType, IntegerType, LongType, FloatType, DoubleType, TimestampType) + + + val findTightestCommonType: (DataType, DataType) => Option[DataType] = { + case (t1, t2) if t1 == t2 => Some(t1) + case (NullType, t1) => Some(t1) + case (t1, NullType) => Some(t1) + case (StringType, t2) => Some(StringType) + case (t1, StringType) => Some(StringType) + + // Promote numeric types to the highest of the two and all numeric types to unlimited decimal + case (t1, t2) if Seq(t1, t2).forall(numericPrecedence.contains) => + val index = numericPrecedence.lastIndexWhere(t => t == t1 || t == t2) + Some(numericPrecedence(index)) + + case _ => None + } +} diff --git a/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/package.scala b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/package.scala new file mode 100644 index 00000000000..d9e3733af60 --- /dev/null +++ b/extensions/spark-excel/src/main/scala/com/webank/wedatasphere/spark/excel/package.scala @@ -0,0 +1,30 @@ +package com.webank.wedatasphere.spark + +import org.apache.poi.ss.usermodel.Row.MissingCellPolicy +import org.apache.poi.ss.usermodel.{Cell, Row} + +package object excel { + + implicit class RichRow(val row: Row) extends AnyVal { + + def eachCellIterator(startColumn: Int, endColumn: Int): Iterator[Option[Cell]] = new Iterator[Option[Cell]] { + private val lastCellInclusive = row.getLastCellNum - 1 + private val endCol = Math.min(endColumn, Math.max(startColumn, lastCellInclusive)) + require(startColumn >= 0 && startColumn <= endCol) + + private var nextCol = startColumn + + override def hasNext: Boolean = nextCol <= endCol && nextCol <= lastCellInclusive + + override def next(): Option[Cell] = { + val next = + if (nextCol > endCol) throw new NoSuchElementException(s"column index = $nextCol") + else Option(row.getCell(nextCol, MissingCellPolicy.RETURN_NULL_AND_BLANK)) + nextCol += 1 + next + } + } + + } + +} diff --git a/gateway/core/pom.xml b/gateway/core/pom.xml index eb0442bb973..5ac68065775 100644 --- a/gateway/core/pom.xml +++ b/gateway/core/pom.xml @@ -22,7 +22,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0-SNAPSHOT + 0.8.0 linkis-gateway-core @@ -52,6 +52,10 @@ websocket-server org.eclipse.jetty.websocket + + io.netty + netty-all + diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewayConfiguration.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewayConfiguration.scala index 9f76d501fa7..f48a15dc362 100644 --- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewayConfiguration.scala +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/config/GatewayConfiguration.scala @@ -29,6 +29,9 @@ object GatewayConfiguration { val PASS_AUTH_REQUEST_URI = CommonVars("wds.linkis.gateway.conf.url.pass.auth", "/dws/").getValue.split(",") + val ENABLE_SSO_LOGIN = CommonVars("wds.linkis.gateway.conf.enable.sso", false) + val SSO_INTERCEPTOR_CLASS = CommonVars("wds.linkis.gateway.conf.sso.interceptor", "") + val ADMIN_USER = CommonVars("wds.linkis.gateway.admin.user", "hadoop") } diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayHttpRequest.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayHttpRequest.scala index 830fca66b7a..104de0509cd 100644 --- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayHttpRequest.scala +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayHttpRequest.scala @@ -32,8 +32,12 @@ trait GatewayHttpRequest { def getHeaders: JMap[String, Array[String]] + def addHeader(headerName: String, headers: Array[String]): Unit + def getQueryParams: JMap[String, Array[String]] + def addCookie(cookieName: String, cookies: Array[Cookie]): Unit + def getCookies: JMap[String, Array[Cookie]] def getRemoteAddress: InetSocketAddress diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayHttpResponse.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayHttpResponse.scala index c31f8ab8daf..c448da8c0b6 100644 --- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayHttpResponse.scala +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/http/GatewayHttpResponse.scala @@ -33,6 +33,8 @@ trait GatewayHttpResponse { def writeWebSocket(message: String): Unit + def redirectTo(url: String): Unit + def sendResponse(): Unit def isCommitted: Boolean diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala index 13986314979..2d459d5228e 100644 --- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/GatewaySSOUtils.scala @@ -17,8 +17,7 @@ package com.webank.wedatasphere.linkis.gateway.security import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} -import com.webank.wedatasphere.linkis.gateway.config.GatewayConfiguration -import com.webank.wedatasphere.linkis.gateway.http.GatewayContext +import com.webank.wedatasphere.linkis.gateway.http.{GatewayContext, GatewayHttpRequest} import com.webank.wedatasphere.linkis.server.exception.LoginExpireException import com.webank.wedatasphere.linkis.server.security.{SSOUtils, ServerSSOUtils} import com.webank.wedatasphere.linkis.server.security.SecurityFilter._ @@ -41,13 +40,12 @@ object GatewaySSOUtils extends Logging { } def getLoginUsername(gatewayContext: GatewayContext): String = SSOUtils.getLoginUsername(getCookies(gatewayContext)) def setLoginUser(gatewayContext: GatewayContext, username: String): Unit = { - if(GatewayConfiguration.ENABLE_PROXY_USER.getValue){ - val proxyUser = ProxyUserUtils.getProxyUser(username) - info(s"switched to proxy user $proxyUser for umUser $username.") - SSOUtils.setLoginUser(c => gatewayContext.getResponse.addCookie(c), proxyUser) - } else { - SSOUtils.setLoginUser(c => gatewayContext.getResponse.addCookie(c), username) - } + val proxyUser = ProxyUserUtils.getProxyUser(username) + SSOUtils.setLoginUser(c => gatewayContext.getResponse.addCookie(c), proxyUser) + } + def setLoginUser(request: GatewayHttpRequest, username: String): Unit = { + val proxyUser = ProxyUserUtils.getProxyUser(username) + SSOUtils.setLoginUser(c => request.addCookie(c.getName, Array(c)), proxyUser) } def removeLoginUser(gatewayContext: GatewayContext): Unit = { SSOUtils.removeLoginUser(gatewayContext.getRequest.getCookies.flatMap(_._2).toArray) diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/ProxyUserUtils.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/ProxyUserUtils.scala index 8f97163b729..c223ebe793e 100644 --- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/ProxyUserUtils.scala +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/ProxyUserUtils.scala @@ -21,29 +21,29 @@ import java.util.concurrent.TimeUnit import com.webank.wedatasphere.linkis.common.utils.{Logging, Utils} import com.webank.wedatasphere.linkis.gateway.config.GatewayConfiguration._ -import com.webank.wedatasphere.linkis.gateway.exception.GatewayErrorException import org.apache.commons.lang.StringUtils object ProxyUserUtils extends Logging { - var props = new Properties + private val props = new Properties if(ENABLE_PROXY_USER.getValue){ Utils.defaultScheduler.scheduleAtFixedRate(new Runnable { override def run(): Unit = { info("loading proxy users.") val newProps = new Properties newProps.load(this.getClass.getResourceAsStream(PROXY_USER_CONFIG.getValue)) - props = newProps + props.clear() + props.putAll(newProps) } }, 0, PROXY_USER_SCAN_INTERVAL.getValue, TimeUnit.MILLISECONDS) } - def getProxyUser(umUser: String): String ={ + def getProxyUser(umUser: String): String = if(ENABLE_PROXY_USER.getValue) { val proxyUser = props.getProperty(umUser) - if(StringUtils.isBlank(proxyUser)){ - throw new GatewayErrorException(10033, "No proxy users available(无可用的代理用户)。"); + if(StringUtils.isBlank(proxyUser)) umUser else { + info(s"switched to proxy user $proxyUser for umUser $umUser.") + proxyUser } - proxyUser - } + } else umUser } diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/SecurityFilter.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/SecurityFilter.scala index 8f7b99aaa0e..e84b6209789 100644 --- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/SecurityFilter.scala +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/SecurityFilter.scala @@ -23,6 +23,7 @@ import com.webank.wedatasphere.linkis.common.conf.Configuration import com.webank.wedatasphere.linkis.common.utils.Utils import com.webank.wedatasphere.linkis.gateway.config.GatewayConfiguration import com.webank.wedatasphere.linkis.gateway.http.GatewayContext +import com.webank.wedatasphere.linkis.gateway.security.sso.SSOInterceptor import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration import com.webank.wedatasphere.linkis.server.exception.{LoginExpireException, NonLoginException} import com.webank.wedatasphere.linkis.server.{Message, validateFailed} @@ -63,10 +64,11 @@ object SecurityFilter { return false } } + val isPassAuthRequest = GatewayConfiguration.PASS_AUTH_REQUEST_URI.exists(gatewayContext.getRequest.getRequestURI.startsWith) if(gatewayContext.getRequest.getRequestURI.startsWith(ServerConfiguration.BDP_SERVER_USER_URI.getValue)) { userRestful.doUserRequest(gatewayContext) false - } else if(GatewayConfiguration.PASS_AUTH_REQUEST_URI.exists(gatewayContext.getRequest.getRequestURI.startsWith)) { + } else if(isPassAuthRequest && !GatewayConfiguration.ENABLE_SSO_LOGIN.getValue) { GatewaySSOUtils.info("No login needed for proxy uri: " + gatewayContext.getRequest.getRequestURI) true } else { @@ -86,6 +88,20 @@ object SecurityFilter { GatewaySSOUtils.info("test mode! login for uri: " + gatewayContext.getRequest.getRequestURI) GatewaySSOUtils.setLoginUser(gatewayContext, testUser) true + } else if(GatewayConfiguration.ENABLE_SSO_LOGIN.getValue) { + val user = SSOInterceptor.getSSOInterceptor.getUser(gatewayContext) + if(StringUtils.isNotBlank(user)) { + GatewaySSOUtils.setLoginUser(gatewayContext.getRequest, user) + true + } else if(isPassAuthRequest) { + gatewayContext.getResponse.redirectTo(SSOInterceptor.getSSOInterceptor.redirectTo(gatewayContext.getRequest.getURI)) + gatewayContext.getResponse.sendResponse() + false + } else { + filterResponse(gatewayContext, Message.noLogin("You are not logged in, please login first(您尚未登录，请先登录)!") + .data("enableSSO", true).data("SSOURL", SSOInterceptor.getSSOInterceptor.redirectTo(gatewayContext.getRequest.getURI)) << gatewayContext.getRequest.getRequestURI) + false + } } else { filterResponse(gatewayContext, Message.noLogin("You are not logged in, please login first(您尚未登录，请先登录)!") << gatewayContext.getRequest.getRequestURI) false diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala index 77e75942b8e..b30cec2dd5d 100644 --- a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/UserRestful.scala @@ -19,6 +19,7 @@ package com.webank.wedatasphere.linkis.gateway.security import com.webank.wedatasphere.linkis.common.utils.{RSAUtils, Utils} import com.webank.wedatasphere.linkis.gateway.config.GatewayConfiguration import com.webank.wedatasphere.linkis.gateway.http.GatewayContext +import com.webank.wedatasphere.linkis.gateway.security.sso.SSOInterceptor import com.webank.wedatasphere.linkis.server.conf.ServerConfiguration import com.webank.wedatasphere.linkis.server.security.SSOUtils import com.webank.wedatasphere.linkis.server.{Message, _} @@ -68,6 +69,7 @@ abstract class AbstractUserRestful extends UserRestful { def logout(gatewayContext: GatewayContext): Message = { GatewaySSOUtils.removeLoginUser(gatewayContext) + if(GatewayConfiguration.ENABLE_SSO_LOGIN.getValue) SSOInterceptor.getSSOInterceptor.logout(gatewayContext) if(resourceReleases != null) resourceReleases.foreach(_.release(gatewayContext)) "Logout successful(退出登录成功)！" } @@ -104,6 +106,7 @@ abstract class UserPwdAbstractUserRestful extends AbstractUserRestful { } else { //warn: For easy to useing linkis,Admin skip login if(GatewayConfiguration.ADMIN_USER.getValue.equals(userName.toString) && userName.toString.equals(password.toString)){ + GatewaySSOUtils.setLoginUser(gatewayContext, userName.toString) "login successful(登录成功)！".data("userName", userName) .data("isAdmin", true) .data("loginNum", 4) diff --git a/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/sso/SSOInterceptor.scala b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/sso/SSOInterceptor.scala new file mode 100644 index 00000000000..5e50654ab6f --- /dev/null +++ b/gateway/core/src/main/scala/com/webank/wedatasphere/linkis/gateway/security/sso/SSOInterceptor.scala @@ -0,0 +1,47 @@ +package com.webank.wedatasphere.linkis.gateway.security.sso + +import java.net.URI + +import com.webank.wedatasphere.linkis.DataWorkCloudApplication +import com.webank.wedatasphere.linkis.common.utils.ClassUtils +import com.webank.wedatasphere.linkis.gateway.config.GatewayConfiguration +import com.webank.wedatasphere.linkis.gateway.http.GatewayContext + +trait SSOInterceptor { + + /** + * 如果打开SSO单点登录功能，当前端跳转SSO登录页面登录成功后，前端再次转发请求给gateway。 + * 用户需实现该接口，通过Request返回user + * @param gatewayContext + * @return + */ + def getUser(gatewayContext: GatewayContext): String + + /** + * 通过前端的requestUrl，用户传回一个可跳转的SSO登录页面URL。 + * 要求：需带上原请求URL，以便登录成功后能跳转回来 + * @param requestUrl + * @return + */ + def redirectTo(requestUrl: URI): String + + /** + * gateway退出时，会调用此接口，以保证gateway清除cookie后，SSO单点登录也会把登录信息清除掉 + * @param gatewayContext + */ + def logout(gatewayContext: GatewayContext): Unit + +} +object SSOInterceptor { + import scala.collection.JavaConversions._ + private var interceptor: SSOInterceptor = _ + def getSSOInterceptor: SSOInterceptor = if(interceptor != null) interceptor else { + val ssoInterceptors = DataWorkCloudApplication.getApplicationContext.getBeansOfType(classOf[SSOInterceptor]) + if(ssoInterceptors != null && !ssoInterceptors.isEmpty) { + interceptor = ssoInterceptors.head._2 + } else { + interceptor = ClassUtils.getClassInstance(GatewayConfiguration.SSO_INTERCEPTOR_CLASS.getValue) + } + interceptor + } +} \ No newline at end of file diff --git a/gateway/gateway-httpclient-support/pom.xml b/gateway/gateway-httpclient-support/pom.xml index d1160f2a96f..2a68a8f9795 100644 --- a/gateway/gateway-httpclient-support/pom.xml +++ b/gateway/gateway-httpclient-support/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0-SNAPSHOT + 0.8.0 linkis-gateway-httpclient-support diff --git a/gateway/gateway-ujes-support/pom.xml b/gateway/gateway-ujes-support/pom.xml index 1bc530642c6..6dbc35f3076 100644 --- a/gateway/gateway-ujes-support/pom.xml +++ b/gateway/gateway-ujes-support/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0-SNAPSHOT + 0.8.0 linkis-gateway-ujes-support diff --git a/gateway/gateway-ujes-support/src/main/resources/log4j.properties b/gateway/gateway-ujes-support/src/main/resources/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/gateway/gateway-ujes-support/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/gateway/springcloudgateway/pom.xml b/gateway/springcloudgateway/pom.xml index 0b898ac4d97..e596d8492ce 100644 --- a/gateway/springcloudgateway/pom.xml +++ b/gateway/springcloudgateway/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0-SNAPSHOT + 0.8.0 linkis-gateway-springcloudgateway @@ -52,12 +52,7 @@ org.springframework.boot spring-boot-starter-reactor-netty 2.0.3.RELEASE - - - io.netty - netty-all - 4.1.17.Final - + io.projectreactor reactor-core diff --git a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala index eb788438e16..06b77a67a54 100644 --- a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala +++ b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/SpringCloudGatewayConfiguration.scala @@ -105,10 +105,6 @@ object SpringCloudGatewayConfiguration extends Logging { val WEBSOCKET_URI = normalPath(ServerConfiguration.BDP_SERVER_SOCKET_URI.getValue) def normalPath(path: String): String = if(path.endsWith("/")) path else path + "/" - val ENABLE_PROXY_USER = CommonVars("wds.linkis.gateway.conf.enable.proxy.user", false) - val PROXY_USER_CONFIG = CommonVars("wds.linkis.gateway.conf.proxy.user.config", "proxy.properties") - val PROXY_USER_SCAN_INTERVAL = CommonVars("wds.linkis.gateway.conf.proxy.user.scan.interval", 1000 * 60 * 10) - def isMergeModuleInstance(serviceId: String): Boolean = serviceId.startsWith(MERGE_MODULE_INSTANCE_HEADER) private val regex = "(\\d+).+".r diff --git a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/http/SpringCloudGatewayHttpRequest.scala b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/http/SpringCloudGatewayHttpRequest.scala index 28592c8d6f0..5dd5a9382d2 100644 --- a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/http/SpringCloudGatewayHttpRequest.scala +++ b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/http/SpringCloudGatewayHttpRequest.scala @@ -18,11 +18,16 @@ package com.webank.wedatasphere.linkis.gateway.springcloud.http import java.net.{InetSocketAddress, URI} +import com.webank.wedatasphere.linkis.gateway.exception.GatewayErrorException import com.webank.wedatasphere.linkis.gateway.http.GatewayHttpRequest import com.webank.wedatasphere.linkis.server._ +import io.netty.handler.codec.http.cookie.DefaultCookie import javax.servlet.http.Cookie import org.apache.commons.lang.StringUtils import org.springframework.http.server.reactive.AbstractServerHttpRequest +import reactor.ipc.netty.http.server.HttpServerRequest + +import scala.collection.JavaConversions /** * created by cooperyang on 2019/1/9. @@ -59,7 +64,9 @@ class SpringCloudGatewayHttpRequest(request: AbstractServerHttpRequest) extends private var requestURI: String = _ private var requestAutowired = false - def setRequestURI(requestURI: String) = this.requestURI = requestURI + def setRequestURI(requestURI: String): Unit = this.requestURI = requestURI + + def getRequest = request override def getRequestURI: String = if(StringUtils.isNotBlank(requestURI)) requestURI else request.getPath.pathWithinApplication.value @@ -67,6 +74,22 @@ class SpringCloudGatewayHttpRequest(request: AbstractServerHttpRequest) extends override def getHeaders: JMap[String, Array[String]] = headers + override def addHeader(headerName: String, headers: Array[String]): Unit = + request.getHeaders.addAll(headerName, JavaConversions.seqAsJavaList(headers.toList)) + + override def addCookie(cookieName: String, cookies: Array[Cookie]): Unit = request.getNativeRequest match { + case httpServerRequest: HttpServerRequest => + httpServerRequest.asInstanceOf[HttpServerRequest].cookies().put(cookieName, JavaConversions.setAsJavaSet(cookies.map { c => + val cookie = new DefaultCookie(c.getName, c.getValue) + cookie.setDomain(c.getDomain) + cookie.setMaxAge(c.getMaxAge) + cookie.setPath(c.getPath) + cookie.setSecure(c.getSecure) + cookie + }.toSet)) + case _ => throw new GatewayErrorException(10040, "Not support method: addCookie in GatewayHttpRequest.") + } + override def getQueryParams: JMap[String, Array[String]] = queryParams override def getCookies: JMap[String, Array[Cookie]] = cookies diff --git a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/http/SpringCloudGatewayHttpResponse.scala b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/http/SpringCloudGatewayHttpResponse.scala index 55617aba466..43d40c873d1 100644 --- a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/http/SpringCloudGatewayHttpResponse.scala +++ b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/http/SpringCloudGatewayHttpResponse.scala @@ -35,6 +35,7 @@ class SpringCloudGatewayHttpResponse(response: ServerHttpResponse) extends Gatew private val cachedHTTPResponseMsg = new StringBuilder private val cachedWebSocketResponseMsg = new StringBuilder + private val cachedRedirectUrlMsg = new StringBuilder private var responseMono: Mono[Void] = _ override def addCookie(cookie: Cookie): Unit = { @@ -55,6 +56,13 @@ class SpringCloudGatewayHttpResponse(response: ServerHttpResponse) extends Gatew override def sendResponse(): Unit = if(responseMono == null) synchronized { if(responseMono != null) return + if(cachedRedirectUrlMsg.nonEmpty) { + if(response.getStatusCode == null || (response.getStatusCode != null && !response.getStatusCode.is3xxRedirection())) + response.setStatusCode(HttpStatus.TEMPORARY_REDIRECT) + response.getHeaders.set("Location", cachedRedirectUrlMsg.toString) + responseMono = response.setComplete() + return + } setHeader("Content-Type", "application/json;charset=UTF-8") if(cachedHTTPResponseMsg.nonEmpty) { val dataBuffer = response.bufferFactory().wrap(cachedHTTPResponseMsg.toString.getBytes(Configuration.BDP_ENCODING.getValue)) @@ -77,7 +85,9 @@ class SpringCloudGatewayHttpResponse(response: ServerHttpResponse) extends Gatew override def isCommitted: Boolean = responseMono != null - def getResponseMono = responseMono + def getResponseMono: Mono[Void] = responseMono override def writeWebSocket(message: String): Unit = cachedWebSocketResponseMsg.append(message) + + override def redirectTo(url: String): Unit = cachedRedirectUrlMsg.append(url) } diff --git a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/websocket/WebsocketGatewayHttpResponse.scala b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/websocket/WebsocketGatewayHttpResponse.scala index bea6a82b309..6b7b0635488 100644 --- a/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/websocket/WebsocketGatewayHttpResponse.scala +++ b/gateway/springcloudgateway/src/main/scala/com/webank/wedatasphere/linkis/gateway/springcloud/websocket/WebsocketGatewayHttpResponse.scala @@ -29,6 +29,7 @@ class WebsocketGatewayHttpResponse extends GatewayHttpResponse { override def setStatus(status: Int): Unit = {} override def write(message: String): Unit = {} override def writeWebSocket(message: String): Unit = cachedWebSocketResponseMsg.append(message) + override def redirectTo(url: String): Unit = {} override def sendResponse(): Unit = {} def getWebSocketMsg: String = cachedWebSocketResponseMsg.toString() override def isCommitted: Boolean = cachedWebSocketResponseMsg.nonEmpty diff --git a/images/linkis-intro-01.png b/images/linkis-intro-01.png new file mode 100644 index 00000000000..60b575dde9f Binary files /dev/null and b/images/linkis-intro-01.png differ diff --git a/images/linkis-intro-02.png b/images/linkis-intro-02.png new file mode 100644 index 00000000000..a31e68182a5 Binary files /dev/null and b/images/linkis-intro-02.png differ diff --git a/images/linkis-intro-03.png b/images/linkis-intro-03.png new file mode 100644 index 00000000000..b999268447a Binary files /dev/null and b/images/linkis-intro-03.png differ diff --git "a/images/\345\212\237\350\203\275\346\250\241\345\235\227\345\233\276.png" "b/images/\345\212\237\350\203\275\346\250\241\345\235\227\345\233\276.png" new file mode 100644 index 00000000000..15dd01fe365 Binary files /dev/null and "b/images/\345\212\237\350\203\275\346\250\241\345\235\227\345\233\276.png" differ diff --git a/pom.xml b/pom.xml index e9d8ee0007c..4c216e3e1b0 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ com.webank.wedatasphere.linkis linkis - 0.5.0 + 0.8.0 pom @@ -70,26 +70,21 @@ - 2.7.2 - + 1.2.1 1.4.4.RELEASE 2.0.3.RELEASE 2.8.5 - 2.9.6 2.11.8 1.8 2.15.2 2.11 2.16 - 2.23.1 9.4.11.v20180605 - 1.7.12 - 3.2.11 - 0.5.0 + 0.8.0 3.3.3 @@ -186,8 +181,21 @@ product + + true + - 0.0.3-SNAPSHOT + 0.8.0 + 4.1.17.Final + 3.5.3 + + + + spark2.1 + + 0.8.0 + 4.0.47.Final + 3.2.11 @@ -242,11 +250,6 @@ maven-site-plugin 3.3 - - org.apache.maven.plugins - maven-deploy-plugin - 2.7 - net.alchim31.maven scala-maven-plugin diff --git a/publicService/application/pom.xml b/publicService/application/pom.xml index f3a1e6d4341..0eca589fd9c 100644 --- a/publicService/application/pom.xml +++ b/publicService/application/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-application @@ -51,7 +51,7 @@ com.webank.wedatasphere.linkis linkis-storage - 0.5.0 + 0.8.0 org.apache.httpcomponents diff --git a/publicService/bin/start-publicservice.sh b/publicService/bin/start-publicservice.sh index 62205219b2c..9c072e7494b 100644 --- a/publicService/bin/start-publicservice.sh +++ b/publicService/bin/start-publicservice.sh @@ -16,7 +16,7 @@ if [[ -f "${DWS_ENGINE_MANAGER_PID}" ]]; then fi export DWS_ENGINE_MANAGER_LOG_PATH=$HOME/logs -export DWS_ENGINE_MANAGER_HEAP_SIZE="5G" +export DWS_ENGINE_MANAGER_HEAP_SIZE="1G" export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m" nohup java $DWS_ENGINE_MANAGER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* com.webank.wedatasphere.linkis.DataWorkCloudApplication 2>&1 > $DWS_ENGINE_MANAGER_LOG_PATH/linkis.out & diff --git a/publicService/conf/linkis.properties b/publicService/conf/linkis.properties index 15bac73f3f7..bd6f7b6bb7d 100644 --- a/publicService/conf/linkis.properties +++ b/publicService/conf/linkis.properties @@ -41,6 +41,10 @@ wds.linkis.workspace.filesystem.localuserrootpath=file:///tmp/linkis/ wds.linkis.workspace.filesystem.hdfsuserrootpath.prefix=hdfs:///tmp/linkis/ wds.linkis.workspace.filesystem.hdfsuserrootpath.suffix=/ +#hadoopconfig +#hadoop.config.dir= +#hive.config.dir= + ##UDF wds.linkis.storage.is.share.node=true wds.linkis.udf.hive.exec.path=/appcom/hive-exec-1.2.1.jar diff --git a/publicService/conf/log4j.properties b/publicService/conf/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/publicService/conf/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/publicService/configuration/pom.xml b/publicService/configuration/pom.xml index fd45ddab73b..f5b5ebbaf2d 100644 --- a/publicService/configuration/pom.xml +++ b/publicService/configuration/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-configuration diff --git a/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/receiver/ConfigurationReceiver.scala b/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/receiver/ConfigurationReceiver.scala index f18477c5afb..a0f41b9fd8f 100644 --- a/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/receiver/ConfigurationReceiver.scala +++ b/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/receiver/ConfigurationReceiver.scala @@ -42,5 +42,5 @@ class ConfigurationReceiver extends Receiver{ case e:RequestQueryAppConfigWithGlobal =>configurationService.queryAppConfigWithGlobal(e.userName,e.creator,e.appName,e.isMerge) } - override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = ??? + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = {} } diff --git a/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/validate/ContainValidator.scala b/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/validate/ContainValidator.scala index c5d38a7116e..9ab30dc53c1 100644 --- a/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/validate/ContainValidator.scala +++ b/publicService/configuration/src/main/scala/com/webank/wedatasphere/linkis/configuration/validate/ContainValidator.scala @@ -20,7 +20,7 @@ package com.webank.wedatasphere.linkis.configuration.validate * Created by allenlliu on 2019/4/8. */ class ContainValidator extends Validator{ - override def validate(value: String, range: String): Boolean = ??? + override def validate(value: String, range: String): Boolean = false override var kind: String = "Contains" } diff --git a/publicService/database/bin/start-database.sh b/publicService/database/bin/start-database.sh index 736e4aea190..89015e422e7 100644 --- a/publicService/database/bin/start-database.sh +++ b/publicService/database/bin/start-database.sh @@ -16,7 +16,7 @@ if [[ -f "${DWS_ENGINE_MANAGER_PID}" ]]; then fi export DWS_ENGINE_MANAGER_LOG_PATH=$HOME/logs -export DWS_ENGINE_MANAGER_HEAP_SIZE="5G" +export DWS_ENGINE_MANAGER_HEAP_SIZE="1G" export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m" nohup java $DWS_ENGINE_MANAGER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* com.webank.wedatasphere.linkis.DataWorkCloudApplication 2>&1 > $DWS_ENGINE_MANAGER_LOG_PATH/linkis-database.out & diff --git a/publicService/database/conf/application.yml b/publicService/database/conf/application.yml index 969d401129e..970e4fa04cb 100644 --- a/publicService/database/conf/application.yml +++ b/publicService/database/conf/application.yml @@ -2,7 +2,7 @@ server: port: 9022 #该模块提供服务的端口（必须） spring: application: - name: cloud-database #模块名，用于做高可用（必须） + name: cloud-datasource #模块名，用于做高可用（必须） #指定eureka Server的地址，用于注册（必须） eureka: diff --git a/publicService/database/conf/linkis.properties b/publicService/database/conf/linkis.properties index 4602b0a4447..cb43d268fcc 100644 --- a/publicService/database/conf/linkis.properties +++ b/publicService/database/conf/linkis.properties @@ -26,7 +26,8 @@ wds.linkis.server.mybatis.datasource.password= wds.linkis.log.clear=true wds.linkis.server.version=v1 -hadoop.config.dir=/appcom/config/hadoop-config +#hadoop.config.dir=/appcom/config/hadoop-config +#hive.config.dir=/appcom/config/hive-config ##restful wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.database.restful.api diff --git a/publicService/database/conf/log4j.properties b/publicService/database/conf/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/publicService/database/conf/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/publicService/database/pom.xml b/publicService/database/pom.xml index f93d5715b2b..c17453b035a 100644 --- a/publicService/database/pom.xml +++ b/publicService/database/pom.xml @@ -21,16 +21,16 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 4.0.0 jar - BDP DataWorkshop Database Project + linkis-database linkis-database - - 1.2.1 - + + + com.webank.wedatasphere.linkis diff --git a/publicService/database/src/main/java/com/webank/wedatasphere/linkis/database/hive/config/HiveDataSourceConfig.java b/publicService/database/src/main/java/com/webank/wedatasphere/linkis/database/hive/config/HiveDataSourceConfig.java index 676303fec64..6589b0ed021 100644 --- a/publicService/database/src/main/java/com/webank/wedatasphere/linkis/database/hive/config/HiveDataSourceConfig.java +++ b/publicService/database/src/main/java/com/webank/wedatasphere/linkis/database/hive/config/HiveDataSourceConfig.java @@ -17,6 +17,7 @@ package com.webank.wedatasphere.linkis.database.hive.config; import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.druid.util.StringUtils; import com.webank.wedatasphere.linkis.common.utils.JavaLog; import com.webank.wedatasphere.linkis.database.util.DWSConfig; import com.webank.wedatasphere.linkis.database.util.HiveUtils; @@ -47,11 +48,16 @@ public DruidDataSource dataSource(){ boolean testWhileIdle = true; boolean testOnBorrow = true; boolean testOnReturn = true; - HiveConf hiveConf = HiveUtils.getDefaultConf(com.webank.wedatasphere.linkis.common.conf.Configuration.HADOOP_ROOT_USER().getValue()); - String url = hiveConf.get("javax.jdo.option.ConnectionURL"); - String username = hiveConf.get("javax.jdo.option.ConnectionUserName"); - String password = hiveConf.get("javax.jdo.option.ConnectionPassword"); - + String url = DWSConfig.HIVE_META_URL.getValue(); + String username = DWSConfig.HIVE_META_USER.getValue(); + String password = DWSConfig.HIVE_META_PASSWORD.getValue(); + if(StringUtils.isEmpty(url) || StringUtils.isEmpty(username) || StringUtils.isEmpty(password)){ + HiveConf hiveConf = HiveUtils.getDefaultConf(com.webank.wedatasphere.linkis.common.conf.Configuration.HADOOP_ROOT_USER().getValue()); + url = hiveConf.get("javax.jdo.option.ConnectionURL"); + username = hiveConf.get("javax.jdo.option.ConnectionUserName"); + password = hiveConf.get("javax.jdo.option.ConnectionPassword"); + info("Database connection address information from hiveConf"); + } DruidDataSource dataSource = new DruidDataSource(); info("Database connection address information =(数据库连接地址信息=)" + url); dataSource.setUrl(url); diff --git a/publicService/database/src/main/java/com/webank/wedatasphere/linkis/database/util/DWSConfig.java b/publicService/database/src/main/java/com/webank/wedatasphere/linkis/database/util/DWSConfig.java index 5a27f8acab6..b51dad295e0 100644 --- a/publicService/database/src/main/java/com/webank/wedatasphere/linkis/database/util/DWSConfig.java +++ b/publicService/database/src/main/java/com/webank/wedatasphere/linkis/database/util/DWSConfig.java @@ -24,8 +24,10 @@ public class DWSConfig { public static CommonVars IDE_URL = CommonVars$.MODULE$.apply("wds.linkis.ide.url", "locahost"); - public static CommonVars HADOOP_CONF_DIR = CommonVars$.MODULE$.apply("hadoop.config.dir", ""); - public static CommonVars HIVE_CONF_DIR = CommonVars$.MODULE$.apply("hive.config.dir", ""); + public static CommonVars HADOOP_CONF_DIR = CommonVars$.MODULE$.apply("hadoop.config.dir", + CommonVars$.MODULE$.apply("HADOOP_CONF_DIR", "/appcom/config/hadoop-config").getValue()); + public static CommonVars HIVE_CONF_DIR = CommonVars$.MODULE$.apply("hive.config.dir", + CommonVars$.MODULE$.apply("HIVE_CONF_DIR", "/appcom/config/hadoop-config").getValue()); public static CommonVars HIVE_META_URL = CommonVars$.MODULE$.apply("hive.meta.url", ""); public static CommonVars HIVE_META_USER = CommonVars$.MODULE$.apply("hive.meta.user", ""); public static CommonVars HIVE_META_PASSWORD = CommonVars$.MODULE$.apply("hive.meta.password", ""); diff --git a/publicService/pom.xml b/publicService/pom.xml index fea4c838843..19471e332f8 100644 --- a/publicService/pom.xml +++ b/publicService/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 4.0.0 @@ -45,7 +45,7 @@ com.webank.wedatasphere.linkis - linkis-cs-udf + linkis-udf ${linkis.version} diff --git a/publicService/query/pom.xml b/publicService/query/pom.xml index 5892739e005..96dbc9c525e 100644 --- a/publicService/query/pom.xml +++ b/publicService/query/pom.xml @@ -24,7 +24,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-query diff --git a/publicService/udf/pom.xml b/publicService/udf/pom.xml index 6f2176689e3..0801c256a0f 100644 --- a/publicService/udf/pom.xml +++ b/publicService/udf/pom.xml @@ -21,13 +21,13 @@ com.webank.wedatasphere.linkis linkis - 0.5.0 + 0.8.0 4.0.0 jar - BDP-UDF udf module - linkis-cs-udf + linkis-udf + linkis-udf UTF-8 diff --git a/publicService/udf/src/main/java/com/webank/wedatasphere/linkis/udf/service/impl/UDFServiceImpl.java b/publicService/udf/src/main/java/com/webank/wedatasphere/linkis/udf/service/impl/UDFServiceImpl.java index b7953cf2861..e0e92dd47aa 100644 --- a/publicService/udf/src/main/java/com/webank/wedatasphere/linkis/udf/service/impl/UDFServiceImpl.java +++ b/publicService/udf/src/main/java/com/webank/wedatasphere/linkis/udf/service/impl/UDFServiceImpl.java @@ -94,7 +94,7 @@ public UDFInfo addUDF(UDFInfo udfInfo, String userName) throws UDFException { private void validateJarFile(UDFInfo udfInfo, String userName) throws UDFException { File targetFile = new File(UdfConfiguration.UDF_TMP_PATH().getValue() + udfInfo.getPath()); FsPath fsPath = new FsPath("file://" + udfInfo.getPath()); - Fs remoteFs = FSFactory.getFsByProxyUser(fsPath, userName); + Fs remoteFs = FSFactory.getFs(fsPath); try{ remoteFs.init(null); if(remoteFs.exists(fsPath)){ @@ -110,23 +110,23 @@ private void validateJarFile(UDFInfo udfInfo, String userName) throws UDFExcepti logger.error(e); throw new UDFException("Verify that there is a problem with the UDF jar package(校验UDF jar包存在问题)：" + e.getMessage()); } - File hiveDependency = new File(UdfConfiguration.UDF_HIVE_EXEC_PATH().getValue()); - String udfClassName = StringUtils.substringBetween(udfInfo.getRegisterFormat(), "\"", "\""); - try{ - URL[] url = {new URL("file://" + targetFile.getAbsolutePath()), new URL("file://" + hiveDependency.getAbsolutePath())}; - URLClassLoader loader = URLClassLoader.newInstance(url); - Class clazz = loader.loadClass(udfClassName); - Constructor constructor = clazz.getConstructor(new Class[0]); - if(!Modifier.isPublic(constructor.getModifiers())) throw new NoSuchMethodException(); - }catch (ClassNotFoundException cne){ - throw new UDFException("There is a problem verifying the UDF jar package: the class is not found(校验UDF jar包存在问题：找不到类) " + cne.getMessage()); - } catch (NoSuchMethodException e) { - throw new UDFException("There is a problem verifying the UDF jar package: class(校验UDF jar包存在问题：类) " + udfClassName + " Missing public no-argument constructor(缺少public的无参数构造方法)"); - }catch (Exception e){ - throw new UDFException("Verify that there is a problem with the UDF jar package(校验UDF jar包存在问题)：" + e.getMessage()); - } finally { - targetFile.delete(); - } +// File hiveDependency = new File(UdfConfiguration.UDF_HIVE_EXEC_PATH().getValue()); +// String udfClassName = StringUtils.substringBetween(udfInfo.getRegisterFormat(), "\"", "\""); +// try{ +// URL[] url = {new URL("file://" + targetFile.getAbsolutePath()), new URL("file://" + hiveDependency.getAbsolutePath())}; +// URLClassLoader loader = URLClassLoader.newInstance(url); +// Class clazz = loader.loadClass(udfClassName); +// Constructor constructor = clazz.getConstructor(new Class[0]); +// if(!Modifier.isPublic(constructor.getModifiers())) throw new NoSuchMethodException(); +// }catch (ClassNotFoundException cne){ +// throw new UDFException("There is a problem verifying the UDF jar package: the class is not found(校验UDF jar包存在问题：找不到类) " + cne.getMessage()); +// } catch (NoSuchMethodException e) { +// throw new UDFException("There is a problem verifying the UDF jar package: class(校验UDF jar包存在问题：类) " + udfClassName + " Missing public no-argument constructor(缺少public的无参数构造方法)"); +// }catch (Exception e){ +// throw new UDFException("Verify that there is a problem with the UDF jar package(校验UDF jar包存在问题)：" + e.getMessage()); +// } finally { +// targetFile.delete(); +// } } @Override diff --git a/publicService/udf/src/main/scala/com/webank/wedatasphere/linkis/udf/utils/UdfConfiguration.scala b/publicService/udf/src/main/scala/com/webank/wedatasphere/linkis/udf/utils/UdfConfiguration.scala index aa76d48a276..40d944f020a 100644 --- a/publicService/udf/src/main/scala/com/webank/wedatasphere/linkis/udf/utils/UdfConfiguration.scala +++ b/publicService/udf/src/main/scala/com/webank/wedatasphere/linkis/udf/utils/UdfConfiguration.scala @@ -21,6 +21,6 @@ import com.webank.wedatasphere.linkis.common.conf.CommonVars object UdfConfiguration { val UDF_HIVE_EXEC_PATH = CommonVars("wds.linkis.udf.hive.exec.path", "/appcom/Install/DataWorkCloudInstall/linkis-linkis-Udf-0.0.3-SNAPSHOT/lib/hive-exec-1.2.1.jar") - val UDF_TMP_PATH = CommonVars("wds.linkis.udf.tmp.path", "/appcom/tmp/dwc/udf/") + val UDF_TMP_PATH = CommonVars("wds.linkis.udf.tmp.path", "/tmp/udf/") } diff --git a/publicService/variable/pom.xml b/publicService/variable/pom.xml index 9f48fefbb09..db6e8b1458d 100644 --- a/publicService/variable/pom.xml +++ b/publicService/variable/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0-SNAPSHOT + 0.8.0 linkis-variable diff --git a/publicService/variable/src/main/scala/com/webank/wedatasphere/linkis/variable/receiver/VariableReceiver.scala b/publicService/variable/src/main/scala/com/webank/wedatasphere/linkis/variable/receiver/VariableReceiver.scala index 86d3e9caa31..258b6d712c0 100644 --- a/publicService/variable/src/main/scala/com/webank/wedatasphere/linkis/variable/receiver/VariableReceiver.scala +++ b/publicService/variable/src/main/scala/com/webank/wedatasphere/linkis/variable/receiver/VariableReceiver.scala @@ -41,5 +41,5 @@ class VariableReceiver extends Receiver{ case e:RequestQueryAppVariable =>variableService.queryAppVariable(e.userName,e.creator,e.appName) } - override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = ??? + override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = {} } diff --git a/publicService/workspace/pom.xml b/publicService/workspace/pom.xml index 6135b2fbf92..caf48c1927d 100644 --- a/publicService/workspace/pom.xml +++ b/publicService/workspace/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-workspace jar diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/conf/WorkSpaceConfiguration.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/conf/WorkSpaceConfiguration.java index 99953c0295e..f95231f18f5 100644 --- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/conf/WorkSpaceConfiguration.java +++ b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/conf/WorkSpaceConfiguration.java @@ -23,7 +23,7 @@ * Created by johnnwang on 2018/11/9. */ public class WorkSpaceConfiguration { - public static final CommonVars LOCAL_USER_ROOT_PATH = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.localuserrootpath","file:///appcom/tmp/dwc/"); + public static final CommonVars LOCAL_USER_ROOT_PATH = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.localuserrootpath","file:///tmp/linkis/"); public static final CommonVars HDFS_USER_ROOT_PATH_PREFIX = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.hdfsuserrootpath.prefix","hdfs:///tmp/"); - public static final CommonVars HDFS_USER_ROOT_PATH_SUFFIX = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.hdfsuserrootpath.suffix","/dwc/"); + public static final CommonVars HDFS_USER_ROOT_PATH_SUFFIX = CommonVars$.MODULE$.apply("wds.linkis.workspace.filesystem.hdfsuserrootpath.suffix","/linkis/"); } diff --git a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/FsRestfulApi.java b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/FsRestfulApi.java index 2477e117d70..6f3e9cccc24 100644 --- a/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/FsRestfulApi.java +++ b/publicService/workspace/src/main/java/com/webank/wedatasphere/linkis/filesystem/restful/api/FsRestfulApi.java @@ -110,10 +110,18 @@ public Response getUserRootPath(@Context HttpServletRequest req,@QueryParam("pat String path = null; String returnType = null; if(pathType.equals("hdfs")){ - path = WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_PREFIX.getValue() + userName + WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_SUFFIX.getValue(); + if (WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_PREFIX.getValue().toString().endsWith("/")){ + path = WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_PREFIX.getValue() + userName + WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_SUFFIX.getValue(); + }else{ + path = WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_PREFIX.getValue() + "/" + userName + WorkSpaceConfiguration.HDFS_USER_ROOT_PATH_SUFFIX.getValue(); + } returnType = "HDFS"; }else { - path = WorkSpaceConfiguration.LOCAL_USER_ROOT_PATH.getValue() + userName + "/"; + if (WorkSpaceConfiguration.LOCAL_USER_ROOT_PATH.getValue().toString().endsWith("/")){ + path = WorkSpaceConfiguration.LOCAL_USER_ROOT_PATH.getValue() + userName + "/"; + }else{ + path = WorkSpaceConfiguration.LOCAL_USER_ROOT_PATH.getValue() + "/" + userName + "/"; + } returnType = "Local"; } FsPath fsPath = new FsPath(path); @@ -261,7 +269,7 @@ public Response getDirFileTrees(@Context HttpServletRequest req, } DirFileTree dirFileTree = new DirFileTree(); dirFileTree.setPath(fsPath.getSchemaPath()); - if(!isInUserWorkspace(path,userName)) throw new WorkSpaceException("The user does not have permission to view the contents of the directory"); + //if(!isInUserWorkspace(path,userName)) throw new WorkSpaceException("The user does not have permission to view the contents of the directory"); if (!fileSystem.canExecute(fsPath) || !fileSystem.canRead(fsPath)) { throw new WorkSpaceException("The user does not have permission to view the contents of the directory(该用户无权限查看该目录的内容)"); } diff --git a/resourceManager/resourcemanagerclient/pom.xml b/resourceManager/resourcemanagerclient/pom.xml index c84d45952a1..fdfd8f92c8a 100644 --- a/resourceManager/resourcemanagerclient/pom.xml +++ b/resourceManager/resourcemanagerclient/pom.xml @@ -20,7 +20,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 0.5.0 + 0.8.0 com.webank.wedatasphere.linkis linkis diff --git a/resourceManager/resourcemanagercommon/pom.xml b/resourceManager/resourcemanagercommon/pom.xml index 073fae17e96..13fc17c5840 100644 --- a/resourceManager/resourcemanagercommon/pom.xml +++ b/resourceManager/resourcemanagercommon/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.5.0 + 0.8.0 com.webank.wedatasphere.linkis linkis diff --git a/resourceManager/resourcemanagerserver/pom.xml b/resourceManager/resourcemanagerserver/pom.xml index 2dea300ca54..fb36ed27fda 100644 --- a/resourceManager/resourcemanagerserver/pom.xml +++ b/resourceManager/resourcemanagerserver/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.5.0 + 0.8.0 com.webank.wedatasphere.linkis linkis @@ -70,10 +70,10 @@ org.ow2.asm asm - - org.jboss.netty - netty - + + + + javax.servlet servlet-api @@ -84,6 +84,22 @@ + + io.netty + netty + 3.6.2.Final + + + org.json4s + json4s-jackson_${scala.binary.version} + 3.2.11 + + + org.scala-lang + scala-library + + + diff --git a/resourceManager/resourcemanagerserver/src/main/assembly/distribution.xml b/resourceManager/resourcemanagerserver/src/main/assembly/distribution.xml index 62c1da8d7e8..89bc2ae0c2e 100644 --- a/resourceManager/resourcemanagerserver/src/main/assembly/distribution.xml +++ b/resourceManager/resourcemanagerserver/src/main/assembly/distribution.xml @@ -144,7 +144,7 @@ commons-lang:commons-lang:jar commons-logging:commons-logging:jar commons-net:commons-net:jar - io.netty:netty:jar + io.netty:netty-all:jar io.netty:netty-buffer:jar io.netty:netty-codec:jar @@ -204,9 +204,9 @@ org.eclipse.jetty:jetty-xml:jar org.fusesource.leveldbjni:leveldbjni-all:jar org.hdrhistogram:HdrHistogram:jar - org.json4s:json4s-ast_2.11:jar - org.json4s:json4s-core_2.11:jar - org.json4s:json4s-jackson_2.11:jar + + + org.jsoup:jsoup:jar org.mortbay.jetty:jetty:jar org.mortbay.jetty:jetty-util:jar diff --git a/resourceManager/resourcemanagerserver/src/main/resources/linkis.properties b/resourceManager/resourcemanagerserver/src/main/resources/linkis.properties index 3400789baaa..cf2a21dd2d2 100644 --- a/resourceManager/resourcemanagerserver/src/main/resources/linkis.properties +++ b/resourceManager/resourcemanagerserver/src/main/resources/linkis.properties @@ -25,5 +25,7 @@ wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.resourcem wds.linkis.server.mybatis.mapperLocations=classpath:com/webank/wedatasphere/linkis/resourcemanager/dao/impl/*.xml wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.linkis.resourcemanager.dao wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.linkis.resourcemanager.dao +#hadoop config dir +#hadoop.config.dir=/appcom/config/hadoop-config diff --git a/resourceManager/resourcemanagerserver/src/main/resources/log4j.properties b/resourceManager/resourcemanagerserver/src/main/resources/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/resourceManager/resourcemanagerserver/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/metric/MetricRMEventExecutor.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/metric/MetricRMEventExecutor.scala index e5cdfd95fc4..b756a325c7a 100644 --- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/metric/MetricRMEventExecutor.scala +++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/metric/MetricRMEventExecutor.scala @@ -57,11 +57,11 @@ class MetricRMEventExecutor(id: Int) extends AbstractExecutor(id) { } - override def getExecutorInfo = ??? + override def getExecutorInfo = null override protected def callback() = { } - override def close(): Unit = ??? + override def close(): Unit = {} } diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventExecutor.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventExecutor.scala index 3be1d45cc18..a00723b6d0d 100644 --- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventExecutor.scala +++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/event/notify/NotifyRMEventExecutor.scala @@ -63,11 +63,11 @@ class NotifyRMEventExecutor(id: Int) extends AbstractExecutor(id) { } } - override def getExecutorInfo = ??? + override def getExecutorInfo = null override protected def callback() = { } - override def close(): Unit = ??? + override def close(): Unit = {} } diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/ModuleListener.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/ModuleListener.scala index 23740df443f..a382ffc40a7 100644 --- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/ModuleListener.scala +++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/ModuleListener.scala @@ -57,7 +57,7 @@ class ModuleListener extends NotifyRMEventListener with Logging { } } - override def onEventError(event: com.webank.wedatasphere.linkis.common.listener.Event, t: scala.Throwable): scala.Unit = ??? + override def onEventError(event: com.webank.wedatasphere.linkis.common.listener.Event, t: scala.Throwable): scala.Unit = {} } diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/UserListener.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/UserListener.scala index 30ca51715b9..d37e89298e6 100644 --- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/UserListener.scala +++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/listener/UserListener.scala @@ -49,7 +49,7 @@ class UserListener extends MetricRMEventListener with Logging { } } - def onEventError(event: Event, t: scala.Throwable): scala.Unit = ??? + def onEventError(event: Event, t: scala.Throwable): scala.Unit = {} } class userEventByUserName extends Comparator[UserSessionStartEvent] { diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventScheduler.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventScheduler.scala index b7065b76d0c..daa15d719d2 100644 --- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventScheduler.scala +++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventScheduler.scala @@ -44,7 +44,7 @@ class EventSchedulerImpl(schedulerContext: SchedulerContext) extends EventSchedu "OK" } - override def start() = ??? + override def start() = {} override def getName = "EventParallelScheduler" diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventSchedulerContextImpl.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventSchedulerContextImpl.scala index aaf9f127951..67b01c03ce7 100644 --- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventSchedulerContextImpl.scala +++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/EventSchedulerContextImpl.scala @@ -58,5 +58,5 @@ class EventSchedulerContextImpl(maxParallelismUsers: Int) extends EventScheduler } } - override def getOrCreateSchedulerListenerBus = ??? + override def getOrCreateSchedulerListenerBus = null } diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/RMEventExecutorManager.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/RMEventExecutorManager.scala index a09441c59c3..d6e14e16c89 100644 --- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/RMEventExecutorManager.scala +++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/schedule/RMEventExecutorManager.scala @@ -44,7 +44,7 @@ class RMEventExecutorManager extends ExecutorManager { this.metricRMEventListenerBus = metricRMEventListenerBus } - override def setExecutorListener(executorListener: ExecutorListener) = ??? + override def setExecutorListener(executorListener: ExecutorListener) = {} override protected def createExecutor(event: SchedulerEvent) = { event match { @@ -88,11 +88,11 @@ class RMEventExecutorManager extends ExecutorManager { } } - override def askExecutor(event: SchedulerEvent, wait: Duration) = ??? + override def askExecutor(event: SchedulerEvent, wait: Duration) = null - override def getById(id: Long) = ??? + override def getById(id: Long) = null - override def getByGroup(groupName: String) = ??? + override def getByGroup(groupName: String) = null override protected def delete(executor: Executor) = { executor.close() diff --git a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/YarnUtil.scala b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/YarnUtil.scala index 859a436c9d9..757b0410a33 100644 --- a/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/YarnUtil.scala +++ b/resourceManager/resourcemanagerserver/src/main/scala/com/webank/wedatasphere/linkis/resourcemanager/utils/YarnUtil.scala @@ -21,7 +21,7 @@ import java.net.ConnectException import com.fasterxml.jackson.core.JsonParseException import com.webank.wedatasphere.linkis.common.utils.{HDFSUtils, Logging, Utils} import com.webank.wedatasphere.linkis.resourcemanager.YarnResource -import com.webank.wedatasphere.linkis.resourcemanager.exception.{RMErrorException, RMWarnException} +import com.webank.wedatasphere.linkis.resourcemanager.exception.{RMErrorException, RMFatalException, RMWarnException} import dispatch.{Http, as} import org.apache.commons.lang.StringUtils import org.apache.hadoop.fs.Path @@ -42,13 +42,14 @@ object YarnUtil extends Logging{ private implicit val executor = ExecutionContext.global private val yarnConf = new YarnConfiguration() private var rm_web_address: String = _ - + private var hadoop_version:String = "2.7.2" implicit val format = DefaultFormats def init() = { yarnConf.addResource(new Path(HDFSUtils.hadoopConfDir, YarnConfiguration.CORE_SITE_CONFIGURATION_FILE)) yarnConf.addResource(new Path(HDFSUtils.hadoopConfDir, YarnConfiguration.YARN_SITE_CONFIGURATION_FILE)) reloadRMWebAddress() + Utils.tryAndErrorMsg(getHadoopVersion())("Failed to get HadoopVersion") } init() private def reloadRMWebAddress() = { @@ -67,6 +68,19 @@ object YarnUtil extends Logging{ info(s"the first, use $rm_web_address to ensure the right rm_web_address.") } } + if(StringUtils.isEmpty(this.rm_web_address)){ + val yarnWebUrl = yarnConf.get("yarn.resourcemanager.webapp.address") + if(StringUtils.isEmpty(yarnWebUrl)) { + val yarnHttps = yarnConf.get("yarn.resourcemanager.webapp.https.address") + if(StringUtils.isEmpty(yarnHttps)){ + throw new RMFatalException(11005,"Cannot find yarn resourcemanager restful address,please to configure yarn-site.xml") + } else { + this.rm_web_address = if(yarnHttps.startsWith("https")) yarnHttps else "https://" + yarnHttps + } + } else{ + this.rm_web_address = if(yarnWebUrl.startsWith("http")) yarnWebUrl else "http://" + yarnWebUrl + } + } } else { info(s"find RM_HA_ID $rmHAId, will try to load the right rm_web_address from HA mode.") yarnConf.set(YarnConfiguration.RM_HA_ID, rmHAId) @@ -78,6 +92,16 @@ object YarnUtil extends Logging{ info(s"Resource Manager WebApp address: $rm_web_address.") } + private def getHadoopVersion():Unit = { + val url = dispatch.url(rm_web_address) / "ws" / "v1" / "cluster" / "info" + val future = Http(url > as.json4s.Json).map {resp => + val resourceManagerVersion = resp \ "clusterInfo" \ "resourceManagerVersion" + info(s"Hadoop version is $resourceManagerVersion") + hadoop_version = resourceManagerVersion.values.asInstanceOf[String] + } + } + + def getQueueInfo(queueName: String): (YarnResource, YarnResource) = { val url = dispatch.url(rm_web_address) / "ws" / "v1" / "cluster" / "scheduler" url.setContentType("application/json", "UTF-8") @@ -92,7 +116,7 @@ object YarnUtil extends Logging{ queue.foreach { q => val yarnQueueName = (q \ "queueName").asInstanceOf[JString].values if(yarnQueueName == realQueueName) return Some(q) - else if(realQueueName.startsWith(yarnQueueName + ".")) return getQueue(q \ "childQueues") + else if(realQueueName.startsWith(yarnQueueName + ".")) return getQueue(getChildQueues(q)) } None case JObject(queue) => @@ -104,10 +128,19 @@ object YarnUtil extends Logging{ } case JNull | JNothing => None } + def getChildQueues(resp:JValue):JValue = if (hadoop_version.startsWith("2.7")) { + resp \ "childQueues" + } else { + resp \ "childQueues" \ "queue" + } + val future = Http(url > as.json4s.Json).map {resp => - val childQueues = resp \ "scheduler" \ "schedulerInfo" \ "rootQueue" \ "childQueues" + val childQueues = getChildQueues(resp \ "scheduler" \ "schedulerInfo" \ "rootQueue") val queue = getQueue(childQueues) - if(queue.isEmpty) throw new RMWarnException(111006, s"queue $queueName is not exists in YARN.") + if(queue.isEmpty) { + debug(s"cannot find any information about queue $queueName, response: " + resp) + throw new RMWarnException(111006, s"queue $queueName is not exists in YARN.") + } (getYarnResource(queue.map( _ \ "maxResources")).get, getYarnResource(queue.map( _ \ "usedResources")).get) } diff --git a/storage/pesIO/io-engine/pom.xml b/storage/pesIO/io-engine/pom.xml index 303c17a43ee..ec8b2b07142 100644 --- a/storage/pesIO/io-engine/pom.xml +++ b/storage/pesIO/io-engine/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.5.0 + 0.8.0 com.webank.wedatasphere.linkis linkis diff --git a/storage/pesIO/io-enginemanager/pom.xml b/storage/pesIO/io-enginemanager/pom.xml index 4d21226cbe1..332e28234d7 100644 --- a/storage/pesIO/io-enginemanager/pom.xml +++ b/storage/pesIO/io-enginemanager/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.5.0 + 0.8.0 com.webank.wedatasphere.linkis linkis diff --git a/storage/pesIO/io-entrance/pom.xml b/storage/pesIO/io-entrance/pom.xml index d6996884370..379e67c4b68 100644 --- a/storage/pesIO/io-entrance/pom.xml +++ b/storage/pesIO/io-entrance/pom.xml @@ -20,7 +20,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - 0.5.0 + 0.8.0 com.webank.wedatasphere.linkis linkis diff --git a/storage/storage/pom.xml b/storage/storage/pom.xml index 06336fef07b..cbc3829275f 100644 --- a/storage/storage/pom.xml +++ b/storage/storage/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.5.0 + 0.8.0 com.webank.wedatasphere.linkis linkis diff --git a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageUtils.scala b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageUtils.scala index c0604d75463..a9e0733a86c 100644 --- a/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageUtils.scala +++ b/storage/storage/src/main/scala/com/webank/wedatasphere/linkis/storage/utils/StorageUtils.scala @@ -21,6 +21,7 @@ import java.lang.reflect.Method import com.webank.wedatasphere.linkis.common.io.{Fs, FsPath} import com.webank.wedatasphere.linkis.common.utils.{HDFSUtils, Logging, Utils} +import com.webank.wedatasphere.linkis.storage.exception.StorageFatalException import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} import com.webank.wedatasphere.linkis.storage.resultset.{ResultSetFactory, ResultSetReader, ResultSetWriter} import org.apache.commons.lang.StringUtils @@ -161,7 +162,10 @@ object StorageUtils extends Logging{ def isHDFSNode:Boolean = { val confPath = new File(HDFSUtils.hadoopConfDir) - if(!confPath.exists() || confPath.isFile) false else true + //TODO IO-client mode need return false + if(!confPath.exists() || confPath.isFile) + throw new StorageFatalException(50001, "HDFS configuration was not read, please configure hadoop.config.dir or add env:HADOOP_CONF_DIR") + else true } /** diff --git a/ujes/client/pom.xml b/ujes/client/pom.xml index 5c5d22cbe45..0a2aea53e3f 100644 --- a/ujes/client/pom.xml +++ b/ujes/client/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0-SNAPSHOT + 0.8.0 linkis-ujes-client diff --git a/ujes/client/src/main/resources/log4j.properties b/ujes/client/src/main/resources/log4j.properties index aeead7f9a6d..178f8dfa260 100644 --- a/ujes/client/src/main/resources/log4j.properties +++ b/ujes/client/src/main/resources/log4j.properties @@ -30,7 +30,7 @@ log4j.appender.com.webank.bdp.ide.core.Threshold=INFO log4j.additivity.com.webank.bdp.ide.core=false log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout log4j.appender.com.webank.bdp.ide.core.Append=true -log4j.appender.com.webank.bdp.ide.core.File=/appcom/Install/tomcat-ide/logs/ide-engine.log +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/ujes/definedEngines/hive/engine/pom.xml b/ujes/definedEngines/hive/engine/pom.xml index 2928b9a651f..c3f2848d636 100644 --- a/ujes/definedEngines/hive/engine/pom.xml +++ b/ujes/definedEngines/hive/engine/pom.xml @@ -21,16 +21,16 @@ linkis com.webank.wedatasphere.linkis - 0.5.0-SNAPSHOT + 0.8.0 4.0.0 linkis-hive-engine - - 1.2.1 - + + + com.webank.wedatasphere.linkis @@ -225,6 +225,10 @@ org.slf4j slf4j-api + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + @@ -247,7 +251,6 @@ org.apache.hadoop hadoop-client 2.7.2 - provided diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala index eb37936f262..a7e57f34bfe 100644 --- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala +++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/HiveEngineSpringConfiguration.scala @@ -40,6 +40,6 @@ class HiveEngineSpringConfiguration { @Bean(Array("engineHooks")) def generateEngineHooks:Array[EngineHook] = { LOG.info("engineHooks are set in hive.") - Array(new ReleaseEngineHook, new MaxExecuteNumEngineHook, new HiveAddJarsEngineHook, new JarUdfEngineHook, new UseDatabaseEngineHook) + Array(new ReleaseEngineHook, new MaxExecuteNumEngineHook, new HiveAddJarsEngineHook, new JarUdfEngineHook) } } diff --git a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala index 1f3f41ee5c5..d4c825a674c 100644 --- a/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala +++ b/ujes/definedEngines/hive/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/hive/executor/HiveEngineExecutor.scala @@ -141,20 +141,20 @@ class HiveEngineExecutor(outputPrintLimit:Int, val startTime = System.currentTimeMillis() try{ //to get hive query plan - val context = new Context(hiveConf) - val parseDriver = new ParseDriver - val tree = ParseUtils.findRootNonNullToken(parseDriver.parse(realCode, context)) - val sem = SemanticAnalyzerFactory.get(hiveConf, tree) - sem.analyze(tree, context) - sem.validate() - val queryPlan = new QueryPlan(realCode, sem, 0L, null, "query") - val numberOfJobs = Utilities.getMRTasks(queryPlan.getRootTasks).size - numberOfMRJobs = numberOfJobs - queryPlan.getRootTasks foreach {task => - LOG.info(s"MR job jobID ${task.getJobID} and job is ${task.getMapWork.size()}") - } - LOG.info(s"$realCode has $numberOfJobs MR jobs to do") - if (numberOfJobs != 0) engineExecutorContext.appendStdout(s"Your hive sql has $numberOfJobs MR jobs to do") +// val context = new Context(hiveConf) +// val parseDriver = new ParseDriver +// val tree = ParseUtils.findRootNonNullToken(parseDriver.parse(realCode, context)) +// val sem = SemanticAnalyzerFactory.get(hiveConf, tree) +// sem.analyze(tree, context) +// sem.validate() +// val queryPlan = new QueryPlan(realCode, sem, 0L, null, "query") +// val numberOfJobs = Utilities.getMRTasks(queryPlan.getRootTasks).size + numberOfMRJobs = 1 +// queryPlan.getRootTasks foreach {task => +// LOG.info(s"MR job jobID ${task.getJobID} and job is ${task.getMapWork.size()}") +// } +// LOG.info(s"$realCode has $numberOfJobs MR jobs to do") +// if (numberOfJobs != 0) engineExecutorContext.appendStdout(s"Your hive sql has $numberOfJobs MR jobs to do") val hiveResponse:CommandProcessorResponse = driver.run(realCode) if (hiveResponse.getResponseCode != 0) { LOG.error("Hive query failed, reason: ", hiveResponse.getException) diff --git a/ujes/definedEngines/hive/enginemanager/bin/start-hiveenginemanager.sh b/ujes/definedEngines/hive/enginemanager/bin/start-hiveenginemanager.sh index 1926b9532f7..c00a4fcf827 100644 --- a/ujes/definedEngines/hive/enginemanager/bin/start-hiveenginemanager.sh +++ b/ujes/definedEngines/hive/enginemanager/bin/start-hiveenginemanager.sh @@ -16,7 +16,7 @@ if [[ -f "${DWS_ENGINE_MANAGER_PID}" ]]; then fi export DWS_ENGINE_MANAGER_LOG_PATH=$HOME/logs -export DWS_ENGINE_MANAGER_HEAP_SIZE="5G" +export DWS_ENGINE_MANAGER_HEAP_SIZE="1G" export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m" nohup java $DWS_ENGINE_MANAGER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* com.webank.wedatasphere.linkis.DataWorkCloudApplication 2>&1 > $DWS_ENGINE_MANAGER_LOG_PATH/linkis.out & diff --git a/ujes/definedEngines/hive/enginemanager/pom.xml b/ujes/definedEngines/hive/enginemanager/pom.xml index e61b446c72e..debc8ffda69 100644 --- a/ujes/definedEngines/hive/enginemanager/pom.xml +++ b/ujes/definedEngines/hive/enginemanager/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0-SNAPSHOT + 0.8.0 4.0.0 diff --git a/ujes/definedEngines/hive/enginemanager/src/main/assembly/distribution.xml b/ujes/definedEngines/hive/enginemanager/src/main/assembly/distribution.xml index a21aa23aed8..7e5fe25138c 100644 --- a/ujes/definedEngines/hive/enginemanager/src/main/assembly/distribution.xml +++ b/ujes/definedEngines/hive/enginemanager/src/main/assembly/distribution.xml @@ -55,7 +55,6 @@ --> antlr:antlr:jar - aopalliance:aopalliance:jar asm:asm:jar cglib:cglib:jar com.amazonaws:aws-java-sdk-autoscaling:jar @@ -113,7 +112,6 @@ commons-cli:commons-cli:jar commons-collections:commons-collections:jar commons-configuration:commons-configuration:jar - commons-daemon:commons-daemon:jar commons-dbcp:commons-dbcp:jar commons-digester:commons-digester:jar commons-io:commons-io:jar @@ -235,7 +233,6 @@ org.jvnet:tiger-types:jar org.latencyutils:LatencyUtils:jar org.mortbay.jasper:apache-el:jar - org.mortbay.jetty:jetty:jar org.mortbay.jetty:jetty-util:jar org.ow2.asm:asm-analysis:jar org.ow2.asm:asm-commons:jar @@ -320,5 +317,4 @@ - - + \ No newline at end of file diff --git a/ujes/definedEngines/hive/enginemanager/src/main/resources/willink-engine.properties b/ujes/definedEngines/hive/enginemanager/src/main/resources/linkis-engine.properties similarity index 95% rename from ujes/definedEngines/hive/enginemanager/src/main/resources/willink-engine.properties rename to ujes/definedEngines/hive/enginemanager/src/main/resources/linkis-engine.properties index a9fb48f8f76..5462a3b894c 100644 --- a/ujes/definedEngines/hive/enginemanager/src/main/resources/willink-engine.properties +++ b/ujes/definedEngines/hive/enginemanager/src/main/resources/linkis-engine.properties @@ -21,5 +21,5 @@ wds.linkis.server.component.exclude.packages=com.webank.wedatasphere.linkis.engi wds.linkis.server.component.exclude.classes=com.webank.wedatasphere.linkis.resourcemanager.service.annotation.RMAnnotationParser wds.linkis.server.version=v1 #hadoop config dir -hadoop.config.dir=/appcom/config/hadoop-config +#hadoop.config.dir=/appcom/config/hadoop-config diff --git a/ujes/definedEngines/hive/enginemanager/src/main/resources/linkis.properties b/ujes/definedEngines/hive/enginemanager/src/main/resources/linkis.properties index eabefd9e661..b7b8d593c80 100644 --- a/ujes/definedEngines/hive/enginemanager/src/main/resources/linkis.properties +++ b/ujes/definedEngines/hive/enginemanager/src/main/resources/linkis.properties @@ -19,6 +19,6 @@ wds.linkis.engine.application.name=hiveEngine wds.linkis.server.component.exclude.packages=com.webank.wedatasphere.linkis.engine.,com.webank.wedatasphere.linkis.udf. wds.linkis.server.version=v1 #hadoop config dir -hadoop.config.dir=/appcom/config/hadoop-config +#hadoop.config.dir=/appcom/config/hadoop-config #sudo script wds.linkis.enginemanager.sudo.script=/appcom/Install/WillinkInstall/linkis-ujes-spark-enginemanager/bin/rootScript.sh \ No newline at end of file diff --git a/ujes/definedEngines/hive/enginemanager/src/main/resources/log4j.properties b/ujes/definedEngines/hive/enginemanager/src/main/resources/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/ujes/definedEngines/hive/enginemanager/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala b/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala index 773afb33930..1a8a358c56c 100644 --- a/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala +++ b/ujes/definedEngines/hive/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/hive/process/HiveQLProcessBuilder.scala @@ -60,7 +60,7 @@ class HiveQLProcessBuilder extends JavaProcessEngineBuilder{ override protected def getExtractClasspath: Array[String] = { if (StringUtils.isNotBlank(HiveEngineConfiguration.HIVE_CLIENT_EXTRACLASSPATH.getValue)){ HiveEngineConfiguration.HIVE_CLIENT_EXTRACLASSPATH.getValue.split(",") - }else Array[String]("/appcom/commonlib/webank_bdp_udf.jar") + }else Array.empty } /** diff --git a/ujes/definedEngines/hive/entrance/bin/start-hiveentrance.sh b/ujes/definedEngines/hive/entrance/bin/start-hiveentrance.sh index f9482866a2e..679cbf6f433 100644 --- a/ujes/definedEngines/hive/entrance/bin/start-hiveentrance.sh +++ b/ujes/definedEngines/hive/entrance/bin/start-hiveentrance.sh @@ -16,7 +16,7 @@ if [[ -f "${DWS_ENGINE_MANAGER_PID}" ]]; then fi export DWS_ENGINE_MANAGER_LOG_PATH=$HOME/logs -export DWS_ENGINE_MANAGER_HEAP_SIZE="5G" +export DWS_ENGINE_MANAGER_HEAP_SIZE="1G" export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m" nohup java $DWS_ENGINE_MANAGER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* com.webank.wedatasphere.linkis.DataWorkCloudApplication 2>&1 > $DWS_ENGINE_MANAGER_LOG_PATH/linkis.out & diff --git a/ujes/definedEngines/hive/entrance/pom.xml b/ujes/definedEngines/hive/entrance/pom.xml index 9e96b0e1a9c..66e56977e37 100644 --- a/ujes/definedEngines/hive/entrance/pom.xml +++ b/ujes/definedEngines/hive/entrance/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0-SNAPSHOT + 0.8.0 4.0.0 diff --git a/ujes/definedEngines/hive/entrance/src/main/resources/linkis.properties b/ujes/definedEngines/hive/entrance/src/main/resources/linkis.properties index 6d787f8e77d..264c7cb0d1a 100644 --- a/ujes/definedEngines/hive/entrance/src/main/resources/linkis.properties +++ b/ujes/definedEngines/hive/entrance/src/main/resources/linkis.properties @@ -26,8 +26,9 @@ wds.linkis.console.config.application.name=cloud-publicservice wds.linkis.engine.creation.wait.time.max=20m wds.linkis.server.version=v1 #hadoop config dir -hadoop.config.dir=/appcom/config/hadoop-config +#hadoop.config.dir=/appcom/config/hadoop-config wds.linkis.entrance.config.logPath=file:///appcom/logs/dataworkcloud/dwc/ +wds.linkis.resultSet.store.path=hdfs:/// wds.linkis.server.socket.mode=true \ No newline at end of file diff --git a/ujes/definedEngines/hive/entrance/src/main/resources/log4j.properties b/ujes/definedEngines/hive/entrance/src/main/resources/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/ujes/definedEngines/hive/entrance/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/ujes/definedEngines/pipeline/engine/pom.xml b/ujes/definedEngines/pipeline/engine/pom.xml index 36bc3c49e16..59f0ccd17ba 100644 --- a/ujes/definedEngines/pipeline/engine/pom.xml +++ b/ujes/definedEngines/pipeline/engine/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.5.0 + 0.8.0 com.webank.wedatasphere.linkis linkis diff --git a/ujes/definedEngines/pipeline/enginemanager/bin/start-pipelineenginemanager.sh b/ujes/definedEngines/pipeline/enginemanager/bin/start-pipelineenginemanager.sh index 98f6e668edc..68686424f32 100644 --- a/ujes/definedEngines/pipeline/enginemanager/bin/start-pipelineenginemanager.sh +++ b/ujes/definedEngines/pipeline/enginemanager/bin/start-pipelineenginemanager.sh @@ -16,7 +16,7 @@ if [[ -f "${DWS_ENGINE_MANAGER_PID}" ]]; then fi export DWS_ENGINE_MANAGER_LOG_PATH=$HOME/logs -export DWS_ENGINE_MANAGER_HEAP_SIZE="5G" +export DWS_ENGINE_MANAGER_HEAP_SIZE="1G" export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m" nohup java $DWS_ENGINE_MANAGER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* com.webank.wedatasphere.linkis.DataWorkCloudApplication 2>&1 > $DWS_ENGINE_MANAGER_LOG_PATH/linkis.out & diff --git a/ujes/definedEngines/pipeline/enginemanager/pom.xml b/ujes/definedEngines/pipeline/enginemanager/pom.xml index 2c011fe4e7a..b3d4c1349df 100644 --- a/ujes/definedEngines/pipeline/enginemanager/pom.xml +++ b/ujes/definedEngines/pipeline/enginemanager/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.5.0 + 0.8.0 com.webank.wedatasphere.linkis linkis @@ -39,7 +39,7 @@ com.webank.wedatasphere.linkis linkis-pipeline-engine - 0.5.0 + 0.8.0 diff --git a/ujes/definedEngines/pipeline/enginemanager/src/main/resources/willink-engine.properties b/ujes/definedEngines/pipeline/enginemanager/src/main/resources/linkis-engine.properties similarity index 89% rename from ujes/definedEngines/pipeline/enginemanager/src/main/resources/willink-engine.properties rename to ujes/definedEngines/pipeline/enginemanager/src/main/resources/linkis-engine.properties index a3dbc5acf42..6a1a445b0cd 100644 --- a/ujes/definedEngines/pipeline/enginemanager/src/main/resources/willink-engine.properties +++ b/ujes/definedEngines/pipeline/enginemanager/src/main/resources/linkis-engine.properties @@ -24,5 +24,6 @@ wds.linkis.server.component.exclude.classes=com.webank.wedatasphere.linkis.resou wds.linkis.console.config.application.name=cloud-publicservice -hadoop.config.dir=/appcom/config/hadoop-config -spark.config.dir=/appcom/config/spark-config \ No newline at end of file +#hadoop.config.dir=/appcom/config/hadoop-config +#spark.config.dir=/appcom/config/spark-config +wds.linkis.server.version=v1 \ No newline at end of file diff --git a/ujes/definedEngines/pipeline/enginemanager/src/main/resources/linkis.properties b/ujes/definedEngines/pipeline/enginemanager/src/main/resources/linkis.properties index 88929b56727..10d1d0900f5 100644 --- a/ujes/definedEngines/pipeline/enginemanager/src/main/resources/linkis.properties +++ b/ujes/definedEngines/pipeline/enginemanager/src/main/resources/linkis.properties @@ -22,8 +22,9 @@ wds.linkis.engine.application.name=pipeLineEngine wds.linkis.server.component.exclude.packages=com.webank.wedatasphere.linkis.engine.,com.webank.wedatasphere.linkis.udf. wds.linkis.console.config.application.name=cloud-publicservice -hadoop.config.dir=/appcom/config/hadoop-config -spark.config.dir=/appcom/config/spark-config +#hadoop.config.dir=/appcom/config/hadoop-config +#spark.config.dir=/appcom/config/spark-config #rootScript shell path -wds.linkis.enginemanager.sudo.script=/appcom/tmp/johnnwang/Install/linkis-ujes-spark-enginemanager/bin/rootScript.sh \ No newline at end of file +wds.linkis.enginemanager.sudo.script=/appcom/tmp/johnnwang/Install/linkis-ujes-spark-enginemanager/bin/rootScript.sh +wds.linkis.server.version=v1 \ No newline at end of file diff --git a/ujes/definedEngines/pipeline/enginemanager/src/main/resources/log4j.properties b/ujes/definedEngines/pipeline/enginemanager/src/main/resources/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/ujes/definedEngines/pipeline/enginemanager/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/ujes/definedEngines/pipeline/enginemanager/src/main/resources/rootScript.sh b/ujes/definedEngines/pipeline/enginemanager/src/main/resources/rootScript.sh new file mode 100644 index 00000000000..c26a2d42da6 --- /dev/null +++ b/ujes/definedEngines/pipeline/enginemanager/src/main/resources/rootScript.sh @@ -0,0 +1,16 @@ +#!/usr/bin/expect -f + +set user [lindex $argv 0] +set command [lindex $argv 1] +set timeout -1 + +spawn sudo su - +expect "~]# " +send "su - $user\r" +expect "~]* " +send "$command \r" +expect "~]* " +send "exit\r" +expect "~]# " +send "exit\r" +expect "~]$ " diff --git a/ujes/definedEngines/pipeline/entrance/bin/start-pipelineentrance.sh b/ujes/definedEngines/pipeline/entrance/bin/start-pipelineentrance.sh index 4733a4e7140..fb30e95a943 100644 --- a/ujes/definedEngines/pipeline/entrance/bin/start-pipelineentrance.sh +++ b/ujes/definedEngines/pipeline/entrance/bin/start-pipelineentrance.sh @@ -16,7 +16,7 @@ if [[ -f "${DWS_ENGINE_MANAGER_PID}" ]]; then fi export DWS_ENGINE_MANAGER_LOG_PATH=$HOME/logs -export DWS_ENGINE_MANAGER_HEAP_SIZE="5G" +export DWS_ENGINE_MANAGER_HEAP_SIZE="1G" export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m" nohup java $DWS_ENGINE_MANAGER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* com.webank.wedatasphere.linkis.DataWorkCloudApplication 2>&1 > $DWS_ENGINE_MANAGER_LOG_PATH/linkis.out & diff --git a/ujes/definedEngines/pipeline/entrance/pom.xml b/ujes/definedEngines/pipeline/entrance/pom.xml index b1a1904833b..39ae10f7637 100644 --- a/ujes/definedEngines/pipeline/entrance/pom.xml +++ b/ujes/definedEngines/pipeline/entrance/pom.xml @@ -21,7 +21,7 @@ 4.0.0 - 0.5.0 + 0.8.0 com.webank.wedatasphere.linkis linkis diff --git a/ujes/definedEngines/pipeline/entrance/src/main/resources/linkis.properties b/ujes/definedEngines/pipeline/entrance/src/main/resources/linkis.properties index c61ed52e011..57570a388e0 100644 --- a/ujes/definedEngines/pipeline/entrance/src/main/resources/linkis.properties +++ b/ujes/definedEngines/pipeline/entrance/src/main/resources/linkis.properties @@ -26,15 +26,14 @@ wds.linkis.console.config.application.name=cloud-publicservice wds.linkis.engine.creation.wait.time.max=20m -wds.linkis.resultSet.store.path=/tmp - +wds.linkis.resultSet.store.path=hdfs:/// +wds.linkis.server.version=v1 wds.linkis.server.socket.mode=true bdp.server.distinct.mode=true -hadoop.config.dir=/appcom/config/hadoop-config -spark.config.dir=/appcom/config/spark-config +#hadoop.config.dir=/appcom/config/hadoop-config +#spark.config.dir=/appcom/config/spark-config -wds.linkis.server.socket.mode=true \ No newline at end of file diff --git a/ujes/definedEngines/pipeline/entrance/src/main/resources/log4j.properties b/ujes/definedEngines/pipeline/entrance/src/main/resources/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/ujes/definedEngines/pipeline/entrance/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/ujes/definedEngines/python/engine/pom.xml b/ujes/definedEngines/python/engine/pom.xml index 34f2c47c9b8..0769ad1a00d 100644 --- a/ujes/definedEngines/python/engine/pom.xml +++ b/ujes/definedEngines/python/engine/pom.xml @@ -23,15 +23,13 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-python-engine ${linkis.version} jar - - 2.1.0 - + diff --git a/ujes/definedEngines/python/engine/src/main/resources/python/python.py b/ujes/definedEngines/python/engine/src/main/resources/python/python.py index 958428f3baf..cefdc2b3ba6 100644 --- a/ujes/definedEngines/python/engine/src/main/resources/python/python.py +++ b/ujes/definedEngines/python/engine/src/main/resources/python/python.py @@ -1,5 +1,5 @@ import os, sys, getopt, traceback, json, re -import resource +#import resource # set memory #memoryLimit = long(sys.argv[3]) @@ -12,12 +12,10 @@ from py4j.java_gateway import java_import, JavaGateway, GatewayClient from py4j.protocol import Py4JJavaError, Py4JNetworkError -import ast import traceback import warnings import signal import base64 -import pandas as pd from py4j.java_gateway import JavaGateway from io import BytesIO try: @@ -118,13 +116,6 @@ def _setup_matplotlib(self): import matplotlib except ImportError: return - # Make sure custom backends are available in the PYTHONPATH - rootdir = os.environ.get('ZEPPELIN_HOME', os.getcwd()) - mpl_path = os.path.join(rootdir, 'interpreter', 'lib', 'python') - if mpl_path not in sys.path: - sys.path.append(mpl_path) - - # Finally check if backend exists, and if so configure as appropriate try: matplotlib.use('module://backend_zinline') import backend_zinline diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala index a378a2b5925..d72bfb68ac5 100644 --- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala +++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/PythonSession.scala @@ -265,13 +265,13 @@ object PythonSession { info("this is pythonPath") val pythonPath = new ArrayBuffer[String] //借用spark的py4j - val pythonHomePath = new File(SPARK_HOME.getValue, "python").getPath + val pythonHomePath = new File(PythonEngineConfiguration.PY4J_HOME.getValue) info(s"pythonHomePath => $pythonHomePath" ) - val pythonParentPath = new File(pythonHomePath, "lib") - info(s"pythonParentPath => $pythonParentPath" ) - pythonPath += pythonHomePath + //val pythonParentPath = new File(pythonHomePath, "lib") + //info(s"pythonParentPath => $pythonParentPath" ) + pythonPath += pythonHomePath.getPath info(s"pythonPath => $pythonPath" ) - pythonParentPath.listFiles(new FileFilter { + pythonHomePath.listFiles(new FileFilter { override def accept(pathname: File): Boolean = pathname.getName.endsWith(".zip") }).foreach(f => pythonPath += f.getPath) pythonPath.mkString(File.pathSeparator) diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/PythonEngineConfiguration.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/PythonEngineConfiguration.scala index 7e3298d72c9..424e0bd94fb 100644 --- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/PythonEngineConfiguration.scala +++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/PythonEngineConfiguration.scala @@ -18,10 +18,12 @@ package com.webank.wedatasphere.linkis.engine.conf import com.webank.wedatasphere.linkis.common.conf.CommonVars + /** * created by allenlliu on 2019/3/21 * Description: */ object PythonEngineConfiguration { val PYTHON_CONSOLE_OUTPUT_LINE_LIMIT = CommonVars("wds.linkis.python.line.limit", 10) + val PY4J_HOME = CommonVars("wds.linkis.python.py4j.path", this.getClass.getResource("/").getPath) } diff --git a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/util/PythonConfiguration.scala b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/util/PythonConfiguration.scala index f3d245fee8b..7da1ce16e76 100644 --- a/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/util/PythonConfiguration.scala +++ b/ujes/definedEngines/python/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/util/PythonConfiguration.scala @@ -8,7 +8,6 @@ import com.webank.wedatasphere.linkis.common.utils.Logging */ object PythonConfiguration extends Logging { - val SPARK_HOME = CommonVars[String]("spark.home", CommonVars[String]("SPARK_HOME", "/appcom/Install/spark").getValue) val PYTHON_SCRIPT:CommonVars[String] = CommonVars[String]("python.script", "python3", "Specify a Python startup script that accepts only shared storage paths（指定Python启动脚本，该路径只接受共享存储的路径）.") val PYTHON_PATH:CommonVars[String] = CommonVars[String]("python.path", "", "Specify Python's extra path, which only accepts shared storage paths（指定Python额外的path，该路径只接受共享存储的路径）.") diff --git a/ujes/definedEngines/python/enginemanager/bin/start-pythonenginemanager.sh b/ujes/definedEngines/python/enginemanager/bin/start-pythonenginemanager.sh index e2f75c3771d..19757138865 100644 --- a/ujes/definedEngines/python/enginemanager/bin/start-pythonenginemanager.sh +++ b/ujes/definedEngines/python/enginemanager/bin/start-pythonenginemanager.sh @@ -16,7 +16,7 @@ if [[ -f "${DWS_ENGINE_MANAGER_PID}" ]]; then fi export DWS_ENGINE_MANAGER_LOG_PATH=$HOME/logs -export DWS_ENGINE_MANAGER_HEAP_SIZE="5G" +export DWS_ENGINE_MANAGER_HEAP_SIZE="1G" export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m" nohup java $DWS_ENGINE_MANAGER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* com.webank.wedatasphere.linkis.DataWorkCloudApplication 2>&1 > $DWS_ENGINE_MANAGER_LOG_PATH/linkis.out & diff --git a/ujes/definedEngines/python/enginemanager/pom.xml b/ujes/definedEngines/python/enginemanager/pom.xml index 49230e78009..bc6fbffc741 100644 --- a/ujes/definedEngines/python/enginemanager/pom.xml +++ b/ujes/definedEngines/python/enginemanager/pom.xml @@ -23,17 +23,17 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-python-enginemanager - 0.5.0 + 0.8.0 jar - 0.5.0 + 0.8.0 @@ -61,24 +61,7 @@ com.webank.wedatasphere.linkis linkis-python-engine ${linkis.version} - - - - - org.apache.spark - spark-hive_2.11 - 2.1.0 - - - slf4j-log4j12 - org.slf4j - - - slf4j-api - org.slf4j - - - + diff --git a/ujes/definedEngines/python/enginemanager/src/main/assembly/distribution.xml b/ujes/definedEngines/python/enginemanager/src/main/assembly/distribution.xml index b1f700809b5..0c31c8f8b6b 100644 --- a/ujes/definedEngines/python/enginemanager/src/main/assembly/distribution.xml +++ b/ujes/definedEngines/python/enginemanager/src/main/assembly/distribution.xml @@ -53,7 +53,7 @@ commons* org.apache.hadoop:* io.netty:netty*--> - antlr:antlr:jar + asm:asm:jar cglib:cglib:jar com.fasterxml.jackson.core:jackson-annotations:jar @@ -80,12 +80,12 @@ commons-collections:commons-collections:jar commons-configuration:commons-configuration:jar commons-daemon:commons-daemon:jar - commons-dbcp:commons-dbcp:jar + commons-digester:commons-digester:jar commons-httpclient:commons-httpclient:jar commons-io:commons-io:jar commons-lang:commons-lang:jar - commons-pool:commons-pool:jar + io.netty:netty-buffer:jar io.netty:netty-codec:jar io.netty:netty-codec-http:jar @@ -99,13 +99,13 @@ io.reactivex:rxnetty-servo:jar javax.servlet:javax.servlet-api:jar javax.servlet.jsp:jsp-api:jar - javax.validation:validation-api:jar - javax.ws.rs:javax.ws.rs-api:jar + + log4j:log4j:jar net.databinder.dispatch:dispatch-core_2.11:jar net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar - org.antlr:antlr-runtime:jar - org.antlr:stringtemplate:jar + + org.apache.commons:commons-compress:jar org.apache.curator:curator-client:jar org.apache.curator:curator-framework:jar @@ -136,13 +136,13 @@ org.eclipse.jetty:jetty-webapp:jar org.eclipse.jetty:jetty-xml:jar org.fusesource.leveldbjni:leveldbjni-all:jar - org.glassfish.hk2.external:javax.inject:jar + org.json4s:json4s-ast_2.11:jar org.json4s:json4s-core_2.11:jar org.json4s:json4s-jackson_2.11:jar diff --git a/ujes/definedEngines/python/enginemanager/src/main/resources/willink-engine.properties b/ujes/definedEngines/python/enginemanager/src/main/resources/linkis-engine.properties similarity index 91% rename from ujes/definedEngines/python/enginemanager/src/main/resources/willink-engine.properties rename to ujes/definedEngines/python/enginemanager/src/main/resources/linkis-engine.properties index e8c0ac78122..7431c0c9c84 100644 --- a/ujes/definedEngines/python/enginemanager/src/main/resources/willink-engine.properties +++ b/ujes/definedEngines/python/enginemanager/src/main/resources/linkis-engine.properties @@ -20,6 +20,6 @@ wds.linkis.engine.application.name=pythonEngine wds.linkis.server.component.exclude.packages=com.webank.wedatasphere.linkis.enginemanager.,com.webank.wedatasphere.linkis.udf. wds.linkis.server.component.exclude.classes=com.webank.wedatasphere.linkis.resourcemanager.service.annotation.RMAnnotationParser python.script=python -hadoop.config.dir=/appcom/config/hadoop-config -spark.config.dir=/appcom/config/spark-config +#hadoop.config.dir=/appcom/config/hadoop-config +#spark.config.dir=/appcom/config/spark-config wds.linkis.server.version=v1 \ No newline at end of file diff --git a/ujes/definedEngines/python/enginemanager/src/main/resources/linkis.properties b/ujes/definedEngines/python/enginemanager/src/main/resources/linkis.properties index d63aff3e8a4..311ce3c9e93 100644 --- a/ujes/definedEngines/python/enginemanager/src/main/resources/linkis.properties +++ b/ujes/definedEngines/python/enginemanager/src/main/resources/linkis.properties @@ -19,7 +19,7 @@ wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.entrance.restful wds.linkis.engine.application.name=pythonEngine wds.linkis.server.component.exclude.packages=com.webank.wedatasphere.linkis.engine.,com.webank.wedatasphere.linkis.udf. -hadoop.config.dir=/appcom/config/hadoop-config -spark.config.dir=/appcom/config/spark-config +#hadoop.config.dir=/appcom/config/hadoop-config +#spark.config.dir=/appcom/config/spark-config wds.linkis.server.version=v1 -wds.linkis.enginemanager.sudo.script=/appcom/tmp/johnnwang/Install/linkis-ujes-spark-enginemanager/bin/rootScript.sh \ No newline at end of file +wds.linkis.enginemanager.sudo.script=rootScript.sh \ No newline at end of file diff --git a/ujes/definedEngines/python/enginemanager/src/main/resources/log4j.properties b/ujes/definedEngines/python/enginemanager/src/main/resources/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/ujes/definedEngines/python/enginemanager/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/ujes/definedEngines/python/enginemanager/src/main/resources/py4j-0.10.4-src.zip b/ujes/definedEngines/python/enginemanager/src/main/resources/py4j-0.10.4-src.zip new file mode 100644 index 00000000000..8c3829e3287 Binary files /dev/null and b/ujes/definedEngines/python/enginemanager/src/main/resources/py4j-0.10.4-src.zip differ diff --git a/ujes/definedEngines/python/enginemanager/src/main/resources/pyspark.zip b/ujes/definedEngines/python/enginemanager/src/main/resources/pyspark.zip new file mode 100644 index 00000000000..1e908576db6 Binary files /dev/null and b/ujes/definedEngines/python/enginemanager/src/main/resources/pyspark.zip differ diff --git a/ujes/definedEngines/python/entrance/bin/start-pythonentrance.sh b/ujes/definedEngines/python/entrance/bin/start-pythonentrance.sh index daadf37ee36..41d5fb3c4f8 100644 --- a/ujes/definedEngines/python/entrance/bin/start-pythonentrance.sh +++ b/ujes/definedEngines/python/entrance/bin/start-pythonentrance.sh @@ -16,7 +16,7 @@ if [[ -f "${DWS_ENGINE_MANAGER_PID}" ]]; then fi export DWS_ENGINE_MANAGER_LOG_PATH=$HOME/logs -export DWS_ENGINE_MANAGER_HEAP_SIZE="5G" +export DWS_ENGINE_MANAGER_HEAP_SIZE="1G" export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m" nohup java $DWS_ENGINE_MANAGER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* com.webank.wedatasphere.linkis.DataWorkCloudApplication 2>&1 > $DWS_ENGINE_MANAGER_LOG_PATH/linkis.out & diff --git a/ujes/definedEngines/python/entrance/pom.xml b/ujes/definedEngines/python/entrance/pom.xml index 3d4758d1429..b847c68fa35 100644 --- a/ujes/definedEngines/python/entrance/pom.xml +++ b/ujes/definedEngines/python/entrance/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-python-entracne diff --git a/ujes/definedEngines/python/entrance/src/main/resources/linkis.properties b/ujes/definedEngines/python/entrance/src/main/resources/linkis.properties index 506036f7079..323ba1e1e19 100644 --- a/ujes/definedEngines/python/entrance/src/main/resources/linkis.properties +++ b/ujes/definedEngines/python/entrance/src/main/resources/linkis.properties @@ -31,4 +31,6 @@ wds.linkis.entrance.config.logPath=file:///appcom/logs/dataworkcloud/dwc/ wds.linkis.server.socket.mode=true -hadoop.config.dir=/appcom/config/hadoop-config \ No newline at end of file +#hadoop.config.dir=/appcom/config/hadoop-config + +wds.linkis.resultSet.store.path=hdfs:/// \ No newline at end of file diff --git a/ujes/definedEngines/python/entrance/src/main/resources/log4j.properties b/ujes/definedEngines/python/entrance/src/main/resources/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/ujes/definedEngines/python/entrance/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/ujes/definedEngines/spark/engine/pom.xml b/ujes/definedEngines/spark/engine/pom.xml index 4fba8a21975..4ef836cb12d 100644 --- a/ujes/definedEngines/spark/engine/pom.xml +++ b/ujes/definedEngines/spark/engine/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0-SNAPSHOT + 0.8.0 linkis-ujes-spark-engine @@ -31,7 +31,7 @@ - 2.1.0 + 2.4.3 diff --git a/ujes/definedEngines/spark/engine/src/main/resources/python/mix_pyspark.py b/ujes/definedEngines/spark/engine/src/main/resources/python/mix_pyspark.py index 177eb3e04ec..4b1a341dfa9 100644 --- a/ujes/definedEngines/spark/engine/src/main/resources/python/mix_pyspark.py +++ b/ujes/definedEngines/spark/engine/src/main/resources/python/mix_pyspark.py @@ -1,6 +1,6 @@ import sys, getopt, traceback, json, re -import matplotlib -matplotlib.use('Agg') +import os +os.environ['PYSPARK_ALLOW_INSECURE_GATEWAY']='1' zipPaths = sys.argv[3] paths = zipPaths.split(':') for i in range(len(paths)): @@ -27,6 +27,27 @@ # for back compatibility from pyspark.sql import SQLContext, HiveContext, Row +def setup_matplotlib(): + # If we don't have matplotlib installed don't bother continuing + try: + import matplotlib + except ImportError: + return + try: + matplotlib.use('module://backend_zinline') + import backend_zinline + + # Everything looks good so make config assuming that we are using + # an inline backend + self.configure_mpl(width=600, height=400, dpi=72, + fontsize=10, interactive=True, format='png') + except ImportError: + # Fall back to Agg if no custom backend installed + matplotlib.use('Agg') + print("Unable to load inline matplotlib backend, " + "falling back to Agg") +setup_matplotlib() + class Logger(object): def __init__(self): self.out = "" @@ -53,6 +74,7 @@ def reset(self): def flush(self): pass + class SparkVersion(object): SPARK_1_4_0 = 140 SPARK_1_3_0 = 130 @@ -66,6 +88,7 @@ def isAutoConvertEnabled(self): def isImportAllPackageUnderSparkSql(self): return self.version >= self.SPARK_1_3_0 + output = Logger() errorOutput = ErrorLogger() sys.stdout = output @@ -137,12 +160,6 @@ def show_matplotlib(p, fmt="png", width="auto", height="auto", **kwargs): intp.showHTML(jobGroup,html.format(width=width, height=height, img=img_str)) img.close() -def rddToDF(rdd, header): - from pyspark import RDD - if isinstance(rdd, RDD): - intp.rddToDF(rdd, header) - else: - raise Exception("rdd is not RDD") def saveDFToCsv(df, path, hasheader=True,isOverwrite=False,option={}): from pyspark.sql import DataFrame @@ -178,10 +195,7 @@ def __init__(self, intp, sqlc): self.sqlc = sqlc def register(self, udfName, udf): self.sqlc.registerFunction(udfName, udf) - def listUDFs(self): - self.intp.listUDFs() - def existsUDF(self, name): - self.intp.existsUDF(name) + udf = UDF(intp, sqlc) intp.onPythonScriptInitialized() diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala index 7f4770b81ed..17bf4907eea 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/Interpreter/PythonInterpreter.scala @@ -58,6 +58,7 @@ object PythonInterpreter { env.put("PYSPARK_GATEWAY_PORT", "" + gatewayServer.getListeningPort) env.put("SPARK_HOME", SparkConfiguration.SPARK_HOME.getValue) + env.put("PYSPARK_ALLOW_INSECURE_GATEWAY" , "1") // builder.redirectError(Redirect.INHERIT) val process = builder.start() diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala index 39535c6df8f..7ef322196f9 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkConfiguration.scala @@ -30,7 +30,7 @@ object SparkConfiguration { val DWC_SPARK_USEHIVECONTEXT = CommonVars[Boolean]("wds.linkis.spark.useHiveContext", true) val SPARK_LANGUAGE_REPL_INIT_TIME = CommonVars[TimeType]("wds.linkis.engine.spark.language-repl.init.time", new TimeType("30s")) val SPARK_REPL_CLASSDIR = CommonVars[String]("spark.repl.classdir", "/tmp", "默认master") - val SPARK_HOME = CommonVars[String]("spark.home","/appcom/Install/spark") + val SPARK_HOME = CommonVars[String]("spark.home", CommonVars[String]("SPARK_HOME", "/appcom/Install/spark").getValue) val PROCESS_MAX_THREADS = CommonVars[Int]("wds.linkis.process.threadpool.max", 100) val SPARK_NF_FRACTION_LENGTH = CommonVars[Int]("wds.linkis.engine.spark.fraction.length", 30) val SPARK_CONSOLE_OUTPUT_NUM = CommonVars[Int]("wds.linkis.spark.output.line.limit", 10) diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala index 1a41557d1d1..06786dd708c 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/configuration/SparkEngineServerSpringConfiguration.scala @@ -1,38 +1,37 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.wedatasphere.linkis.engine.configuration - -import com.webank.wedatasphere.linkis.engine.condition.EngineHooksCondition -import com.webank.wedatasphere.linkis.engine.execute.hook._ -import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, SparkCombinedCodeParser} -import com.webank.wedatasphere.linkis.tispark.engine.hook.UserDataBaseHook -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean -import org.springframework.context.annotation.{Bean, Conditional, Configuration} - -/** - * Created by allenlliu on 2018/12/3. - */ -@Configuration -class SparkEngineServerSpringConfiguration { - @Bean(Array("codeParser")) - def createCodeParser(): CodeParser = new SparkCombinedCodeParser() - - - @Bean(Array("engineHooks")) - @Conditional(Array(classOf[EngineHooksCondition])) - def createEngineHooks(): Array[EngineHook] = Array(new ReleaseEngineHook, new MaxExecuteNumEngineHook, new JarUdfEngineHook, new PyUdfEngineHook, new ScalaUdfEngineHook, new PyFunctionEngineHook, new ScalaFunctionEngineHook,new UserDataBaseHook) -} +/* + * Copyright 2019 WeBank + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.webank.wedatasphere.linkis.engine.configuration + +import com.webank.wedatasphere.linkis.engine.condition.EngineHooksCondition +import com.webank.wedatasphere.linkis.engine.execute.hook._ +import com.webank.wedatasphere.linkis.engine.execute.{CodeParser, EngineHook, SparkCombinedCodeParser} +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean +import org.springframework.context.annotation.{Bean, Conditional, Configuration} + +/** + * Created by allenlliu on 2018/12/3. + */ +@Configuration +class SparkEngineServerSpringConfiguration { + @Bean(Array("codeParser")) + def createCodeParser(): CodeParser = new SparkCombinedCodeParser() + + + @Bean(Array("engineHooks")) + @Conditional(Array(classOf[EngineHooksCondition])) + def createEngineHooks(): Array[EngineHook] = Array(new ReleaseEngineHook, new MaxExecuteNumEngineHook, new JarUdfEngineHook, new PyUdfEngineHook, new ScalaUdfEngineHook, new PyFunctionEngineHook, new ScalaFunctionEngineHook) +} diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala index 57722627832..22a1483b6e4 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkEngineExecutor.scala @@ -25,7 +25,7 @@ import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration import com.webank.wedatasphere.linkis.engine.exception.{NoSupportEngineException, SparkEngineException} import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineExecutorContext} import com.webank.wedatasphere.linkis.engine.spark.common._ -import com.webank.wedatasphere.linkis.engine.spark.utils.EngineUtils +import com.webank.wedatasphere.linkis.engine.spark.utils.{EngineUtils, JobProgressUtil} import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo import com.webank.wedatasphere.linkis.resourcemanager.{DriverAndYarnResource, LoadInstanceResource, YarnResource} import com.webank.wedatasphere.linkis.scheduler.exception.DWCJobRetryException @@ -38,8 +38,6 @@ import com.webank.wedatasphere.linkis.storage.{LineMetaData, LineRecord} import org.apache.commons.lang.StringUtils import org.apache.spark.SparkContext import org.apache.spark.sql.DataFrame -import org.apache.spark.ui.jobs.JobProgressListener -import org.scalactic.Pass import scala.collection.mutable.ArrayBuffer @@ -48,7 +46,6 @@ import scala.collection.mutable.ArrayBuffer */ class SparkEngineExecutor(val sc: SparkContext, id: Long, outputPrintLimit: Int, sparkExecutors: Seq[SparkExecutor]) extends EngineExecutor(outputPrintLimit, false) with SingleTaskOperateSupport with SingleTaskInfoSupport with Logging { - private val jobListener = sc.getClass.getDeclaredMethod("jobProgressListener").invoke(sc).asInstanceOf[JobProgressListener] val queryNum = new AtomicLong(0) private var jobGroup: String = _ @@ -134,7 +131,7 @@ class SparkEngineExecutor(val sc: SparkContext, id: Long, outputPrintLimit: Int, } info(s"Start to structure sparksql resultset") response - }(Pass) + }{} override protected def executeCompletely(engineExecutorContext: EngineExecutorContext, code: String, completedLine: String): ExecuteResponse = { val newcode = completedLine + code @@ -152,7 +149,7 @@ class SparkEngineExecutor(val sc: SparkContext, id: Long, outputPrintLimit: Int, engineExecutorContext } - override def pause(): Boolean = ??? + override def pause(): Boolean = false override def kill(): Boolean = { if (!sc.isStopped) { @@ -163,58 +160,19 @@ class SparkEngineExecutor(val sc: SparkContext, id: Long, outputPrintLimit: Int, } } - override def resume(): Boolean = ??? + override def resume(): Boolean = false - override def progress(): Float = if (jobGroup == null) 0 + override def progress(): Float =if (jobGroup == null || engineExecutorContext.getTotalParagraph == 0) 0 else { - var totalTask = 0 - var runTask = 0 - val activeJobs = jobListener.activeJobs - val completedJobs = jobListener.completedJobs - activeJobs.values.foreach(f => { - if (f.jobGroup.getOrElse("").equals(jobGroup)) { - runTask = runTask + f.numCompletedTasks - totalTask = totalTask + f.numTasks - } - }) - completedJobs.foreach(f => { - if (f.jobGroup.getOrElse("").equals(jobGroup)) { - runTask = runTask + f.numCompletedTasks + f.numKilledTasks + f.numSkippedTasks - totalTask = totalTask + f.numTasks - } - }) - info("Current JobGroup " + jobGroup + " progress is runTask:" + runTask + ",totalTask:" + totalTask) - if (engineExecutorContext.getTotalParagraph != 0 && totalTask != 0) { - val res = (engineExecutorContext.getCurrentParagraph * 1f - 1f) / engineExecutorContext.getTotalParagraph + (runTask * 1f / totalTask) / engineExecutorContext.getTotalParagraph - info("Current progress is " + res) - res - } else if (engineExecutorContext.getTotalParagraph != 0) { - val res2 = (engineExecutorContext.getCurrentParagraph * 1f - 1f) / engineExecutorContext.getTotalParagraph - info("Current progress is " + res2) - res2 - } else { - 0 - } + (engineExecutorContext.getCurrentParagraph * 1f - 1f )/ engineExecutorContext.getTotalParagraph + JobProgressUtil.progress(sc,jobGroup)/engineExecutorContext.getTotalParagraph } override def getProgressInfo: Array[JobProgressInfo] = if (jobGroup == null) Array.empty else { info("request new progress info for jobGroup is " + jobGroup) - var progressInfoArray = ArrayBuffer[JobProgressInfo]() - val activeJobs = jobListener.activeJobs - val completedJobs = jobListener.completedJobs - activeJobs.values.foreach(f => { - if (f.jobGroup.getOrElse("").equals(jobGroup)) { - val progress = JobProgressInfo(f.jobGroup.getOrElse(""), f.numTasks, f.numActiveTasks, f.numFailedTasks, f.numCompletedTasks + f.numSkippedTasks) - progressInfoArray += progress - } - }) - completedJobs.foreach(f => { - if (f.jobGroup.getOrElse("").equals(jobGroup)) { - val progress = JobProgressInfo(f.jobGroup.getOrElse(""), f.numTasks, f.numActiveTasks, f.numFailedTasks, f.numCompletedTasks + f.numSkippedTasks) - progressInfoArray += progress - } - }) + val progressInfoArray = ArrayBuffer[JobProgressInfo]() + progressInfoArray ++= JobProgressUtil.getActiveJobProgressInfo(sc,jobGroup) + progressInfoArray ++= JobProgressUtil.getCompletedJobProgressInfo(sc,jobGroup) progressInfoArray.toArray } @@ -222,7 +180,7 @@ class SparkEngineExecutor(val sc: SparkContext, id: Long, outputPrintLimit: Int, "" } - override def close(): Unit = ??? + override def close(): Unit = {} } object SQLSession extends Logging { diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala index 1be0718f079..f6a9bc1c93c 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkPythonExecutor.scala @@ -38,7 +38,6 @@ import org.apache.commons.exec.CommandLine import org.apache.commons.lang.StringUtils import org.apache.spark.api.java.JavaSparkContext import org.apache.spark.rdd.RDD -import org.apache.spark.sql.execution.datasources.csv.{RddToDFUtil, UDF} import org.apache.spark.sql.{DataFrame, SQLContext, SparkSession} import org.apache.spark.{SparkContext, SparkException} import py4j.GatewayServer @@ -256,16 +255,11 @@ class SparkPythonExecutor(val sc: SparkContext, val sqlContext: SQLContext,sess info("Pyspark showHTML execute success!") } - def rddToDF(tokenRdd: RDD[Array[String]], header: Array[String]) = RddToDFUtil.rddToDF(sparkSession, tokenRdd, header) - import scala.collection.JavaConverters._ def saveDFToCsv(df: Any, path: String, hasHeader: Boolean = true , isOverwrite: Boolean = false, option: util.Map[String, Any] = new util.HashMap()): Boolean ={ CsvRelation.saveDFToCsv(sparkSession,df.asInstanceOf[DataFrame],path, hasHeader,isOverwrite, option.asScala.toMap) } - def listUDFs() = UDF.listUDFs - - def existsUDF(name: String) = UDF.existsUDF(name) } diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkScalaExecutor.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkScalaExecutor.scala index 56dc14d8c5c..f5fba553d88 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkScalaExecutor.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkScalaExecutor.scala @@ -62,7 +62,7 @@ class SparkScalaExecutor(val sparkConf: SparkConf) extends SparkExecutor{ var outputDir: File = _ protected implicit val executor = Utils.newCachedExecutionContext(5, "Spark-Scala-REPL-Thread-", true) @throws(classOf[IOException]) - override def open = ??? + override def open = {} @@ -249,16 +249,7 @@ class SparkScalaExecutor(val sparkConf: SparkConf) extends SparkExecutor{ sparkILoop.processLine("def showAlias(df: Any, alias:String): Unit = showDF(sparkContext, jobGroup.toString, df, alias,10000, engineExecutorContext.getEngineExecutorContext)") sparkILoop.processLine("def show(df: Any): Unit = showDF(sparkContext, jobGroup.toString, df,\"\",10000,engineExecutorContext.getEngineExecutorContext)") sparkILoop.processLine("def showHtml(content: Any): Unit = showHTML(sparkContext, jobGroup.toString, content, engineExecutorContext.getEngineExecutorContext)") - sparkILoop.processLine("import org.apache.spark.sql.execution.datasources.csv._") - sparkILoop.processLine("def rddToDF(tokenRdd: org.apache.spark.rdd.RDD[Array[String]],header: Array[String]) = RddToDFUtil.rddToDF(spark,tokenRdd,header)") - sparkILoop.processLine("def createTempView(tableName:String,hdfsPath:String,separator: String = \",\") " + - "= RddToDFUtil.createTempView(spark,tableName,hdfsPath,separator)") - sparkILoop.processLine("def createDashBoardView(tableName:String,hdfsPath:String,inferSchema: Boolean = false) " + - "= RddToDFUtil.createTempView(spark,tableName,hdfsPath,isIdeResult = true,inferSchema = inferSchema)") - sparkILoop.processLine("def clearExpireTable(tableName:String) = RddToDFUtil.clearExpireTable(spark,tableName)") sparkILoop.processLine("import org.apache.spark.sql.UDFRegistration") - sparkILoop.processLine("val udf = UDF") - sparkILoop.processLine("implicit def toUDFMethod(udf: UDF.type): UDFRegistration = sqlContext.udf") sparkILoop.processLine("implicit val sparkSession = spark") bindFlag = true warn(s"init sparkILoop cost ${System.currentTimeMillis() - startTime}.") diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkSqlExecutor.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkSqlExecutor.scala index 17bda456891..30ce4063700 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkSqlExecutor.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/executors/SparkSqlExecutor.scala @@ -76,8 +76,8 @@ class SparkSqlExecutor(val sc: SparkContext,val sqlContext: SQLContext) extends override def kind: Kind = SparkSQL() - override def open: Unit = ??? + override def open: Unit = {} - override def close: Unit = ??? + override def close: Unit = {} } diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala index 7f33b6b529f..f7442364514 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/factory/SparkEngineExecutorFactory.scala @@ -32,6 +32,7 @@ import org.apache.spark.{SparkConf, SparkContext} import org.springframework.stereotype.Component import com.google.common.base.Joiner import com.webank.wedatasphere.linkis.common.conf.{CommonVars, TimeType} +import com.webank.wedatasphere.linkis.engine.configuration.SparkConfiguration import org.apache.spark.util.SparkUtils import scala.collection.mutable.ArrayBuffer @@ -50,7 +51,7 @@ class SparkEngineExecutorFactory extends EngineExecutorFactory with Logging{ val conf = new SparkConf(true).setAppName(options.getOrDefault("spark.app.name", "dwc-spark-apps")) val master = conf.getOption("spark.master").getOrElse(CommonVars("spark.master", "yarn").getValue) info(s"------ Create new SparkContext {$master} -------") - val pysparkBasePath = EngineUtils.sparkHome + val pysparkBasePath = SparkConfiguration.SPARK_HOME.getValue val pysparkPath = new File(pysparkBasePath, "python" + File.separator + "lib") val pythonLibUris = pysparkPath.listFiles().map(_.toURI.toString).filter(_.endsWith(".zip")) if (pythonLibUris.length == 2) { diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala index 5e2034316ed..4db7eb8e840 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/EngineUtils.scala @@ -39,7 +39,6 @@ object EngineUtils { private val user:String = System.getProperty("user.name") private var sparkVersion: String = _ private var fileSystem : com.webank.wedatasphere.linkis.common.io.Fs = _ - val sparkHome = CommonVars("spark.home", "").getValue def getName:String = Sender.getThisServiceInstance.getInstance @@ -53,7 +52,7 @@ object EngineUtils { if(sparkVersion != null) { return sparkVersion } - val sparkSubmit = CommonVars("wds.linkis.server.spark-submit", "spark.submit").getValue + val sparkSubmit = CommonVars("wds.linkis.server.spark-submit", "spark-submit").getValue val pb = new ProcessBuilder(sparkSubmit, "--version") pb.redirectErrorStream(true) pb.redirectInput(ProcessBuilder.Redirect.PIPE) diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/JobProgressUtil.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/JobProgressUtil.scala new file mode 100644 index 00000000000..372a95a6c89 --- /dev/null +++ b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/spark/utils/JobProgressUtil.scala @@ -0,0 +1,60 @@ +package com.webank.wedatasphere.linkis.engine.spark.utils + +import com.webank.wedatasphere.linkis.protocol.engine.JobProgressInfo +import org.apache.spark.{JobExecutionStatus, SparkContext, SparkJobInfo} + +/** + * Created by johnnwang on 2019/5/9. + */ +object JobProgressUtil { + def progress(sc: SparkContext, jobGroup : String):Float = { + val jobIds = sc.statusTracker.getJobIdsForGroup(jobGroup) + val jobs = jobIds.flatMap { id => sc.statusTracker.getJobInfo(id) } + val stages = jobs.flatMap { job => + job.stageIds().flatMap(sc.statusTracker.getStageInfo) + } + + val taskCount = stages.map(_.numTasks).sum + val completedTaskCount = stages.map(_.numCompletedTasks).sum + if (taskCount == 0) { + 0f + } else { + (100 * completedTaskCount.toDouble / taskCount).toFloat + } + } + + def getActiveJobProgressInfo(sc:SparkContext,jobGroup : String):Array[JobProgressInfo] = { + val jobIds = sc.statusTracker.getJobIdsForGroup(jobGroup) + val activeJobs = jobIds.flatMap { id => sc.statusTracker.getJobInfo(id) }.filter(_.status() == JobExecutionStatus.RUNNING) + val progressInfos = activeJobs.map { job => + getJobProgressInfoByStages(job, sc, jobGroup) + } + progressInfos + } + + def getCompletedJobProgressInfo(sc:SparkContext,jobGroup : String):Array[JobProgressInfo] = { + val jobIds = sc.statusTracker.getJobIdsForGroup(jobGroup) + val completedJobs = jobIds.flatMap { id => sc.statusTracker.getJobInfo(id) }.filter(_.status() == JobExecutionStatus.SUCCEEDED) + val progressInfos = completedJobs.map { job => + getJobProgressInfoByStages(job, sc, jobGroup) + } + progressInfos + } + + private def getJobProgressInfoByStages(job:SparkJobInfo, sc:SparkContext, jobGroup : String) : JobProgressInfo = { + val stages = job.stageIds().flatMap(sc.statusTracker.getStageInfo) + + var numTasks = 0 + var numActiveTasks = 0 + var numFailedTasks = 0 + + stages.foreach{stageInfo => + numTasks += stageInfo.numTasks() + numActiveTasks += stageInfo.numActiveTasks() + numFailedTasks += stageInfo.numFailedTasks() + } + val numSucceedTasks = numTasks - numActiveTasks - numFailedTasks + JobProgressInfo(getJobId(job.jobId(), jobGroup), numTasks, numActiveTasks, numFailedTasks, numSucceedTasks) + } + private def getJobId( jobId : Int , jobGroup : String ): String = "jobId-" + jobId + "(" + jobGroup + ")" +} diff --git a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/UserDataBaseHook.scala b/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/UserDataBaseHook.scala deleted file mode 100644 index e85ef86f986..00000000000 --- a/ujes/definedEngines/spark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/UserDataBaseHook.scala +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.webank.wedatasphere.linkis.tispark.engine.hook - -import com.webank.wedatasphere.linkis.common.utils.Logging -import com.webank.wedatasphere.linkis.engine.exception.EngineErrorException -import com.webank.wedatasphere.linkis.engine.execute.{EngineExecutor, EngineHook} -import com.webank.wedatasphere.linkis.scheduler.executer.{ExecuteRequest, RunTypeExecuteRequest} -import com.webank.wedatasphere.linkis.server.JMap - -/** - * Created by allenlliu on 2019/4/22. - */ -class UserDataBaseHook extends EngineHook with Logging { - self => - protected var creator: String = _ - protected var user: String = _ - protected var initSpecialCode: String = _ - val runType = "sql" - - @scala.throws[EngineErrorException] - override def beforeCreateEngine(params: JMap[String, String]) = { - creator = params.get("creator") - user = params.get("user") - initSpecialCode = "use " + user + "_ind;\n" - params - } - - @scala.throws[EngineErrorException] - override def afterCreatedEngine(executor: EngineExecutor) = { - info("Set default database for user , code: " + initSpecialCode) - executor.execute(new ExecuteRequest with RunTypeExecuteRequest { - override val code: String = initSpecialCode - override val runType: String = self.runType - }) - info("executed code: " + initSpecialCode) - } -} diff --git a/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/RddToDFUtil.scala b/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/RddToDFUtil.scala deleted file mode 100644 index 323898cb157..00000000000 --- a/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/RddToDFUtil.scala +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources.csv - -import java.math.BigDecimal -import java.sql.Timestamp - -import com.webank.wedatasphere.linkis.common.utils.Logging -import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.util.DateTimeUtils -import org.apache.spark.sql.{DataFrame, Row, SparkSession} -import org.apache.spark.sql.types._ - -import scala.collection.mutable.ArrayBuffer -import scala.util.control.Exception._ - -/** - * Created by allenlliu on 10/16/17. - */ -object RddToDFUtil extends Logging { - - def rddToDF(spark: SparkSession, - tokenRdd: RDD[Array[String]], - header: Array[String], - options: Map[String, String] = Map.empty[String, String]): DataFrame = { - val csvOptions = new CSVOptions(options) - val rows: Array[Array[String]] = tokenRdd.take(25) - val step = rows.length / 5 - var dataTypes: Array[DataType] = null - for (i <- 0 until(rows.length, step)) { - val row = rows(i) - val types = row.map( - att => if("null".equalsIgnoreCase(att)){ - CSVInferSchema.inferField(NullType, null, csvOptions) - } else { - CSVInferSchema.inferField(NullType, att, csvOptions) - } - ) - for (i <- types.indices) { - val newType = types(i) - val oldType = if (dataTypes != null && i < dataTypes.length) dataTypes(i) else null - if (null != oldType && newType != oldType) { - if (newType == StringType || oldType == StringType) { - types(i) = StringType - } - else if (newType == DoubleType || oldType == DoubleType) { - types(i) = DoubleType - } else if (newType == LongType || oldType == LongType) { - types(i) = LongType - }else if (newType == NullType) { - types(i) = oldType - } - } - } - dataTypes = types - } - var count = -1 - val fields: Array[StructField] = dataTypes.map(data => { - count = count + 1 - StructField(header(count), data) - }) - val structType: StructType = StructType(fields) - val rowRdd = tokenRdd.map(att => { - val row = ArrayBuffer[Any]() - for (i <- dataTypes.indices) { - val filed = if ((allCatch opt att(i)).isDefined) { - att(i) - } else { - warn("row is Error:" + att(0)) - "NULL" - } - val data = if("null".equalsIgnoreCase(filed)){ - null - } else if(dataTypes(i) != StringType && filed.isEmpty) { - null - } else{ - dataTypes(i) match { - case StringType => filed - case DoubleType => filed.toDouble - case IntegerType => filed.toInt - case LongType => filed.toLong - case NullType => filed - case BooleanType => filed.toBoolean - case _: DecimalType => new BigDecimal(filed) - case TimestampType => if ((allCatch opt csvOptions.timestampFormat.parse(filed)).isDefined) { - new Timestamp(csvOptions.timestampFormat.parse(filed).getTime) - } else if ((allCatch opt DateTimeUtils.stringToTime(filed)).isDefined) { - new Timestamp(DateTimeUtils.stringToTime(filed).getTime) - } - case other: DataType => - throw new UnsupportedOperationException(s"Unexpected data type $other") - } - } - row += data - } - Row.fromSeq(row) - }) - //warn("row is Error2:" + rowRdd.take(1)) - val df = spark.createDataFrame(rowRdd, structType) - df - } - - def createTempView(spark: SparkSession, - tableName: String, - hdfsPath: String, - separator: String = "\t", - isIdeResult: Boolean = false, - inferSchema: Boolean = false - ): Unit = { - if (spark.sessionState.catalog.getTempView(tableName).isEmpty) { - - val df: DataFrame = if (!isIdeResult) { - spark.read.format("csv") - .option("sep", separator) - .option("inferSchema", true) - .option("header", "true") - .load(hdfsPath) - } else { - val resultRdd = spark.sparkContext.textFile(hdfsPath) - val colRdd = resultRdd.take(2) - val col = colRdd(1).split(separator) - val rowRdd = resultRdd.mapPartitionsWithIndex((index,iter)=> if(index == 0) iter.drop(2) else iter).map(_.split(separator)) - if (!inferSchema) { - val schema = new StructType(col.map(field => StructField(field, StringType))) - val rows = rowRdd.map(att => { - val row = ArrayBuffer[Any]() - for (i <- col.indices) { - val filed = if ((allCatch opt att(i)).isDefined) { - att(i) - } else { - warn("row is Error:" + att(0)) - "NULL" - } - row += filed - } - Row.fromSeq(row) - }) - spark.createDataFrame(rows, schema) - } else { - RddToDFUtil.rddToDF(spark, rowRdd, col, Map("inferSchema" -> "true")) - } - } - df.createOrReplaceTempView(tableName) - } - } - - def clearExpireTable(spark: SparkSession, - tableName: String): Unit ={ - if (spark.sessionState.catalog.getTempView(tableName).isDefined && spark.sqlContext.isCached(tableName)){ - spark.sqlContext.uncacheTable(tableName) - } - } -} diff --git a/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UDF.scala b/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UDF.scala deleted file mode 100644 index 274a973625e..00000000000 --- a/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/sql/execution/datasources/csv/UDF.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2019 WeBank - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.execution.datasources.csv - -import org.apache.spark.sql.SparkSession - -/** - * Created by allenlliu on 2018/12/3. - */ -object UDF extends Serializable{ - - def listUDFs()(implicit spark: SparkSession): Unit = - spark.sessionState.functionRegistry.listFunction().foreach(println) - - def existsUDF(name: String)(implicit spark: SparkSession): Boolean = spark.sessionState.functionRegistry.functionExists(name) -} diff --git a/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/util/SparkUtils.scala b/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/util/SparkUtils.scala index b47021a7302..9662ce6e437 100644 --- a/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/util/SparkUtils.scala +++ b/ujes/definedEngines/spark/engine/src/main/scala/org/apache/spark/util/SparkUtils.scala @@ -35,6 +35,10 @@ object SparkUtils { // def getUserJars(conf : SparkConf, isShell : Boolean) = Utils.getUserJars(conf) - def unionFileLists(leftList: Option[String], rightList: Option[String]) - = Utils.unionFileLists(leftList, rightList) + def unionFileLists(leftList: Option[String], rightList: Option[String]): Set[String] = { + var allFiles = Set[String]() + leftList.foreach { value => allFiles ++= value.split(",") } + rightList.foreach { value => allFiles ++= value.split(",") } + allFiles.filter { _.nonEmpty } + } } diff --git a/ujes/definedEngines/spark/enginemanager/bin/start-sparkenginemanager.sh b/ujes/definedEngines/spark/enginemanager/bin/start-sparkenginemanager.sh index 348b9bcaddb..d2bd1a10e1e 100644 --- a/ujes/definedEngines/spark/enginemanager/bin/start-sparkenginemanager.sh +++ b/ujes/definedEngines/spark/enginemanager/bin/start-sparkenginemanager.sh @@ -16,7 +16,7 @@ if [[ -f "${DWS_ENGINE_MANAGER_PID}" ]]; then fi export DWS_ENGINE_MANAGER_LOG_PATH=$HOME/logs -export DWS_ENGINE_MANAGER_HEAP_SIZE="5G" +export DWS_ENGINE_MANAGER_HEAP_SIZE="1G" export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m" nohup java $DWS_ENGINE_MANAGER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* com.webank.wedatasphere.linkis.DataWorkCloudApplication 2>&1 > $DWS_ENGINE_MANAGER_LOG_PATH/linkis.out & diff --git a/ujes/definedEngines/spark/enginemanager/pom.xml b/ujes/definedEngines/spark/enginemanager/pom.xml index c4287c7d8ec..6bc4c0dae74 100644 --- a/ujes/definedEngines/spark/enginemanager/pom.xml +++ b/ujes/definedEngines/spark/enginemanager/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-ujes-spark-enginemanager @@ -53,21 +53,6 @@ linkis-ujes-engine ${linkis.version} - - org.apache.spark - spark-hive_2.11 - 2.1.0 - - - slf4j-log4j12 - org.slf4j - - - slf4j-api - org.slf4j - - - org.apache.commons commons-exec diff --git a/ujes/definedEngines/spark/enginemanager/src/main/assembly/distribution.xml b/ujes/definedEngines/spark/enginemanager/src/main/assembly/distribution.xml index e6ed3ada2c5..99f18449862 100644 --- a/ujes/definedEngines/spark/enginemanager/src/main/assembly/distribution.xml +++ b/ujes/definedEngines/spark/enginemanager/src/main/assembly/distribution.xml @@ -55,7 +55,6 @@ io.netty:netty* --> - antlr:antlr:jar asm:asm:jar cglib:cglib:jar com.fasterxml.jackson.core:jackson-annotations:jar @@ -86,12 +85,10 @@ commons-collections:commons-collections:jar commons-configuration:commons-configuration:jar commons-daemon:commons-daemon:jar - commons-dbcp:commons-dbcp:jar commons-digester:commons-digester:jar commons-httpclient:commons-httpclient:jar commons-io:commons-io:jar commons-lang:commons-lang:jar - commons-pool:commons-pool:jar io.netty:netty-buffer:jar io.netty:netty-codec:jar io.netty:netty-codec-http:jar @@ -111,8 +108,6 @@ log4j:log4j:jar net.databinder.dispatch:dispatch-core_2.11:jar net.databinder.dispatch:dispatch-json4s-jackson_2.11:jar - org.antlr:antlr-runtime:jar - org.antlr:stringtemplate:jar org.apache.commons:commons-compress:jar org.apache.curator:curator-client:jar org.apache.curator:curator-framework:jar @@ -183,10 +178,11 @@ org.springframework:spring-expression:jar org.springframework:spring-jcl:jar org.springframework:spring-web:jar - org.tukaani:xz:jar + org.yaml:snakeyaml:jar xerces:xercesImpl:jar xmlenc:xmlenc:jar + org.apache.spark:spark* diff --git a/ujes/definedEngines/spark/enginemanager/src/main/resources/willink-engine.properties b/ujes/definedEngines/spark/enginemanager/src/main/resources/linkis-engine.properties similarity index 92% rename from ujes/definedEngines/spark/enginemanager/src/main/resources/willink-engine.properties rename to ujes/definedEngines/spark/enginemanager/src/main/resources/linkis-engine.properties index b36ac76c6ac..58af0f2678f 100644 --- a/ujes/definedEngines/spark/enginemanager/src/main/resources/willink-engine.properties +++ b/ujes/definedEngines/spark/enginemanager/src/main/resources/linkis-engine.properties @@ -23,5 +23,5 @@ wds.linkis.enginemanager.sudo.script=/appcom/tmp/johnnwang/Install/linkis-ujes-s wds.linkis.server.version=v1 wds.linkis.console.config.application.name=cloud-publicservice wds.linkis.engine.udf.app.name=cloud-publicservice -hadoop.config.dir=/appcom/config/hadoop-config -spark.config.dir=/appcom/config/spark-config \ No newline at end of file +#hadoop.config.dir=/appcom/config/hadoop-config +#spark.config.dir=/appcom/config/spark-config \ No newline at end of file diff --git a/ujes/definedEngines/spark/enginemanager/src/main/resources/linkis.properties b/ujes/definedEngines/spark/enginemanager/src/main/resources/linkis.properties index f439089f247..f7a64abd74f 100644 --- a/ujes/definedEngines/spark/enginemanager/src/main/resources/linkis.properties +++ b/ujes/definedEngines/spark/enginemanager/src/main/resources/linkis.properties @@ -22,7 +22,5 @@ wds.linkis.server.version=v1 wds.linkis.console.config.application.name=cloud-publicservice wds.linkis.engine.udf.app.name=cloud-publicservice wds.linkis.enginemanager.sudo.script=/appcom/tmp/johnnwang/Install/linkis-ujes-spark-enginemanager/bin/rootScript.sh -wds.linkis.enginemanager.core.jar=/appcom/tmp/johnnwang/Install/linkis-ujes-spark-enginemanager/lib/linkis-ujes-spark-engine-0.5.0.jar -wds.linkis.spark.driver.conf.mainjar=/appcom/tmp/johnnwang/Install/linkis-ujes-spark-enginemanager/conf:/appcom/tmp/johnnwang/Install/linkis-ujes-spark-enginemanager/lib/* -hadoop.config.dir=/appcom/config/hadoop-config -spark.config.dir=/appcom/config/spark-config \ No newline at end of file +wds.linkis.spark.driver.conf.mainjar= +#spark.config.dir=/appcom/config/spark-config \ No newline at end of file diff --git a/ujes/definedEngines/spark/enginemanager/src/main/resources/log4j.properties b/ujes/definedEngines/spark/enginemanager/src/main/resources/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/ujes/definedEngines/spark/enginemanager/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala index da0ab6f2cd2..f875008a464 100644 --- a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala +++ b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/configuration/SparkConfiguration.scala @@ -18,8 +18,9 @@ package com.webank.wedatasphere.linkis.enginemanager.configuration import com.webank.wedatasphere.linkis.common.conf.{CommonVars, Configuration} -import com.webank.wedatasphere.linkis.common.utils.Logging +import com.webank.wedatasphere.linkis.common.utils.{ClassUtils, Logging} import com.webank.wedatasphere.linkis.enginemanager.AbstractEngineCreator +import com.webank.wedatasphere.linkis.enginemanager.process.{JavaProcessEngineBuilder, SparkSubmitProcessBuilder} import scala.collection.mutable.ArrayBuffer /** @@ -59,12 +60,14 @@ object SparkConfiguration extends Logging { val SPARK_OUTPUTDIR = CommonVars[String]("spark.outputDir", "/home/georgeqiao", "Default output path（默认输出路径）") val DWC_SPARK_USEHIVECONTEXT = CommonVars[Boolean]("wds.linkis.spark.useHiveContext", true) - val ENGINEMANAGER_JAR = CommonVars[String]("wds.linkis.enginemanager.core.jar", "") + val ENGINEMANAGER_JAR = CommonVars[String]("wds.linkis.enginemanager.core.jar", ClassUtils.jarOfClass(classOf[SparkSubmitProcessBuilder]).head) val SPARK_DRIVER_CLASSPATH = CommonVars[String]("wds.linkis.spark.driver.conf.mainjar", "") val SPARK_DRIVER_EXTRA_JAVA_OPTIONS = CommonVars[String]("spark.driver.extraJavaOptions", "\"-Dwds.linkis.configuration=linkis-engine.properties " + getJavaRemotePort + "\"") val DEFAULT_JAVA_OPTS = CommonVars[String]("wds.linkis.engine.javaOpts.default", "-server -XX:+UseG1GC -XX:MaxPermSize=250m -XX:PermSize=128m " + "-Xloggc:%s -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Dwds.linkis.configuration=linkis-engine.properties") val SPARK_ML_BUCKET_FIELDS = CommonVars[String]("wds.linkis.engine.spark.ml.bucketFields", "age[0,18,30,60,100]") + + val SPARK_SUBMIT_CMD = CommonVars[String]("wds.linkis.engine.spark.submit.cmd", "spark-submit") private var Ports: ArrayBuffer[Int] = _ def getJavaRemotePort = { diff --git a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala index 08741449746..04d0266209a 100644 --- a/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala +++ b/ujes/definedEngines/spark/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/SparkSubmitProcessBuilder.scala @@ -41,7 +41,7 @@ class SparkSubmitProcessBuilder extends ProcessEngineBuilder with Logging { protected var port: Int = _ protected var request: RequestEngine = _ protected var userEngineResource: UserEngineResource = _ - private[this] var _executable: Path = AbsolutePath("/appcom/Install/spark-cmd/bin/spark-submit") + private[this] var _executable: Path = AbsolutePath(SPARK_SUBMIT_CMD.getValue) private[this] var _master: Option[String] = None private[this] var _deployMode: Option[String] = None private[this] var _className: Option[String] = None @@ -129,10 +129,6 @@ class SparkSubmitProcessBuilder extends ProcessEngineBuilder with Logging { override def build(engineRequest: EngineResource, request: RequestEngine): Unit = { this.request = request userEngineResource = engineRequest.asInstanceOf[UserEngineResource] - val sparkhome = SPARK_HOME.getValue(request.properties) - if (StringUtils.isEmpty(sparkhome)) { - warn("We cannot find the spark home, use spark-submit to run application.") - } val darResource: DriverAndYarnResource = engineRequest.getResource.asInstanceOf[DriverAndYarnResource] val properties = request.properties this.master("yarn") diff --git a/ujes/definedEngines/spark/entrance/bin/start-sparkentrance.sh b/ujes/definedEngines/spark/entrance/bin/start-sparkentrance.sh index 447dd54b804..eb6bbd02d58 100644 --- a/ujes/definedEngines/spark/entrance/bin/start-sparkentrance.sh +++ b/ujes/definedEngines/spark/entrance/bin/start-sparkentrance.sh @@ -16,13 +16,13 @@ if [[ -f "${DWS_ENGINE_MANAGER_PID}" ]]; then fi export DWS_ENGINE_MANAGER_LOG_PATH=$HOME/logs -export DWS_ENGINE_MANAGER_HEAP_SIZE="5G" +export DWS_ENGINE_MANAGER_HEAP_SIZE="1G" export DWS_ENGINE_MANAGER_JAVA_OPTS="-Xms$DWS_ENGINE_MANAGER_HEAP_SIZE -Xmx$DWS_ENGINE_MANAGER_HEAP_SIZE -XX:+UseG1GC -XX:MaxPermSize=500m" nohup java $DWS_ENGINE_MANAGER_JAVA_OPTS -cp $HOME/conf:$HOME/lib/* com.webank.wedatasphere.linkis.DataWorkCloudApplication 2>&1 > $DWS_ENGINE_MANAGER_LOG_PATH/linkis.out & pid=$! -{} + if [[ -z "${pid}" ]]; then echo "Spark Entrance start failed!" diff --git a/ujes/definedEngines/spark/entrance/pom.xml b/ujes/definedEngines/spark/entrance/pom.xml index a32ceb08245..4c6101c80ce 100644 --- a/ujes/definedEngines/spark/entrance/pom.xml +++ b/ujes/definedEngines/spark/entrance/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-ujes-spark-entracne diff --git a/ujes/definedEngines/spark/entrance/src/main/resources/linkis.properties b/ujes/definedEngines/spark/entrance/src/main/resources/linkis.properties index 5740c45a281..48dfa712943 100644 --- a/ujes/definedEngines/spark/entrance/src/main/resources/linkis.properties +++ b/ujes/definedEngines/spark/entrance/src/main/resources/linkis.properties @@ -27,9 +27,11 @@ wds.linkis.engine.creation.wait.time.max=20m wds.linkis.console.variable.application.name=cloud-publicservice wds.linkis.server.version=v1 -hadoop.config.dir=/appcom/config/hadoop-config -spark.config.dir=/appcom/config/spark-config +#hadoop.config.dir=/appcom/config/hadoop-config +#spark.config.dir=/appcom/config/spark-config wds.linkis.entrance.config.logPath=file:///appcom/logs/dataworkcloud/dwc/ -wds.linkis.server.socket.mode=true \ No newline at end of file +wds.linkis.server.socket.mode=true + +wds.linkis.resultSet.store.path=hdfs:/// \ No newline at end of file diff --git a/ujes/definedEngines/spark/entrance/src/main/resources/log4j.properties b/ujes/definedEngines/spark/entrance/src/main/resources/log4j.properties new file mode 100644 index 00000000000..178f8dfa260 --- /dev/null +++ b/ujes/definedEngines/spark/entrance/src/main/resources/log4j.properties @@ -0,0 +1,36 @@ +# +# Copyright 2019 WeBank +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +### set log levels ### + +log4j.rootCategory=INFO,console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.Threshold=INFO +log4j.appender.console.layout=org.apache.log4j.PatternLayout +#log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n +log4j.appender.console.layout.ConversionPattern= %d{ISO8601} %-5p (%t) %p %c{1} - %m%n + + +log4j.appender.com.webank.bdp.ide.core=org.apache.log4j.DailyRollingFileAppender +log4j.appender.com.webank.bdp.ide.core.Threshold=INFO +log4j.additivity.com.webank.bdp.ide.core=false +log4j.appender.com.webank.bdp.ide.core.layout=org.apache.log4j.PatternLayout +log4j.appender.com.webank.bdp.ide.core.Append=true +log4j.appender.com.webank.bdp.ide.core.File=logs/linkis.log +log4j.appender.com.webank.bdp.ide.core.layout.ConversionPattern= %d{ISO8601} %-5p (%t) [%F:%M(%L)] - %m%n + +log4j.logger.org.springframework=INFO \ No newline at end of file diff --git a/ujes/definedEngines/tispark/pom.xml b/ujes/definedEngines/tispark/engine/pom.xml similarity index 97% rename from ujes/definedEngines/tispark/pom.xml rename to ujes/definedEngines/tispark/engine/pom.xml index 9467b958d9c..6ae49f1e995 100644 --- a/ujes/definedEngines/tispark/pom.xml +++ b/ujes/definedEngines/tispark/engine/pom.xml @@ -21,7 +21,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 4.0.0 diff --git a/ujes/definedEngines/tispark/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala b/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala similarity index 100% rename from ujes/definedEngines/tispark/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala rename to ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/configuration/TiSparkSpringConfiguration.scala diff --git a/ujes/definedEngines/tispark/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala b/ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala similarity index 100% rename from ujes/definedEngines/tispark/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala rename to ujes/definedEngines/tispark/engine/src/main/scala/com/webank/wedatasphere/linkis/tispark/engine/hook/TiSparkHook.scala diff --git a/ujes/engine/pom.xml b/ujes/engine/pom.xml index 19162cacdc9..6fe311d94b3 100644 --- a/ujes/engine/pom.xml +++ b/ujes/engine/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-ujes-engine @@ -57,13 +57,13 @@ com.webank.wedatasphere.linkis - linkis-cs-udf + linkis-udf ${linkis.version} junit junit - RELEASE + 4.12 test diff --git a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala index 482a119ff33..0d32009df7f 100644 --- a/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala +++ b/ujes/engine/src/main/scala/com/webank/wedatasphere/linkis/engine/conf/EngineConfiguration.scala @@ -43,7 +43,7 @@ object EngineConfiguration { val ENGINE_LOG_SEND_TIME_INTERVAL = CommonVars("wds.linkis.engine.log.send.time.interval", 3) - val TMP_PATH = CommonVars("wds.linkis.dataworkclod.engine.tmp.path","file:///appcom/tmp/") + val TMP_PATH = CommonVars("wds.linkis.dataworkclod.engine.tmp.path","file:///tmp/") val ENGINE_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.engine.application.name", "") diff --git a/ujes/enginemanager/pom.xml b/ujes/enginemanager/pom.xml index b26ed181001..0707634a216 100644 --- a/ujes/enginemanager/pom.xml +++ b/ujes/enginemanager/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-ujes-enginemanager @@ -36,7 +36,7 @@ com.webank.wedatasphere.linkis - linkis-cs-udf + linkis-udf com.webank.wedatasphere.linkis @@ -59,6 +59,12 @@ com.webank.wedatasphere.linkis linkis-storage ${linkis.version} + + + com.monitorjbl + xlsx-streamer + + diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EngineManagerConfiguration.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EngineManagerConfiguration.scala index b7f0ad4f8f6..e893d546681 100644 --- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EngineManagerConfiguration.scala +++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EngineManagerConfiguration.scala @@ -31,5 +31,5 @@ object EngineManagerConfiguration { val UNKNOWN_ENGINE_SCAN_INTERVAL = CommonVars("wds.linkis.engine.unknown.scan.interval", new TimeType("2m")) //清除失去掌控的Engine val ENGINE_UDF_APP_NAME = CommonVars("wds.linkis.engine.udf.app.name", "cloud-publicservice") val ENGINE_UDF_BUILT_IN_PATH = CommonVars("wds.linkis.engine.udf.app.name", "/commonlib/webank_bdp_udf.jar") - val ENGINE_UDF_BUFFER = CommonVars("wds.linkis.engine.udf.buffer", "/appcom/tmp/dataworkcloud/udf/") + val ENGINE_UDF_BUFFER = CommonVars("wds.linkis.engine.udf.buffer", "/tmp/udf/") } diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala index 837dc2b3e35..e01d27d370c 100644 --- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala +++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/conf/EnvConfiguration.scala @@ -32,7 +32,6 @@ object EnvConfiguration { val HADOOP_CONF_DIR = CommonVars[String]("hadoop.config.dir", CommonVars[String]("HADOOP_CONF_DIR", "/appcom/config/hadoop-config").getValue) val HBASE_HOME = CommonVars[String]("hbase.home", CommonVars[String]("HBASE_HOME", "/appcom/Install/hbase").getValue) val HBASE_CONF_DIR = CommonVars[String]("hbase.config.dir", CommonVars[String]("HBASE_CONF_DIR", "/appcom/config/hbase-config").getValue) - val SPARK_HOME = CommonVars[String]("spark.home", CommonVars[String]("SPARK_HOME", "/appcom/Install/spark").getValue) val SPARK_CONF_DIR = CommonVars[String]("spark.config.dir", CommonVars[String]("SPARK_CONF_DIR", "/appcom/config/spark-config").getValue) val HIVE_CONF_DIR = CommonVars[String]("hive.config.dir", CommonVars[String]("HIVE_CONF_DIR", "/appcom/config/hive-config").getValue) @@ -49,6 +48,8 @@ object EnvConfiguration { val ENGINE_CLIENT_MEMORY = CommonVars[ByteType]("wds.linkis.engine.client.memory", new ByteType("2g"), "指定所有Engine客户端的默认内存大小") val ENGINE_CLIENT_EXTRACLASSPATH = CommonVars[String]("wds.linkis.engine.client.extraClassPath", "", "指定用户自定义的jar包全路径（多个以英文,分隔）。") + val HADOOP_LIB_NATIVE = CommonVars[String]("linkis.hadoop.lib.native","/appcom/Install/hadoop/lib/native") + private val LOG4J_PROPERTIES_FILE = CommonVars[String]("wds.linkis.engine.log4j.properties.file", "engine-log4j.properties") private val LOG_PATH_FOR_GC_FILE = CommonVars[String]("wds.linkis.engine.gclog.path", "/appcom/logs/dataworkcloud/") private val LOG_PATH_FOR_ENGINE_GROUP = "dataworkcloud.engine.group" diff --git a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/JavaProcessEngineBuilder.scala b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/JavaProcessEngineBuilder.scala index 20a9cd9a4d3..338511265ff 100644 --- a/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/JavaProcessEngineBuilder.scala +++ b/ujes/enginemanager/src/main/scala/com/webank/wedatasphere/linkis/enginemanager/process/JavaProcessEngineBuilder.scala @@ -76,7 +76,7 @@ abstract class JavaProcessEngineBuilder extends ProcessEngineBuilder { } val classpath = getClasspath(request.properties, getExtractClasspath) classpathCheck(classpath) - commandLine += "-Djava.library.path=/appcom/Install/hadoop/lib/native" + commandLine += "-Djava.library.path=" + HADOOP_LIB_NATIVE.getValue commandLine += "-cp" commandLine += classpath.mkString(":") commandLine += "com.webank.wedatasphere.linkis.engine.DataWorkCloudEngineApplication" diff --git a/ujes/entrance/pom.xml b/ujes/entrance/pom.xml index 4fee827bf7d..5c30a424a72 100644 --- a/ujes/entrance/pom.xml +++ b/ujes/entrance/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-ujes-entrance @@ -43,12 +43,18 @@ com.webank.wedatasphere.linkis linkis-storage ${linkis.version} + + + com.monitorjbl + xlsx-streamer + + junit junit - RELEASE + 4.12 test diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala index 721d7114b81..99b6e986310 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/interceptor/impl/CustomVariableUtils.scala @@ -21,7 +21,7 @@ import java.text.SimpleDateFormat import java.util import java.util.{Calendar, Date} -import com.sun.jdi.FloatValue +//import com.sun.jdi.FloatValue import com.webank.wedatasphere.linkis.common.utils.Logging import com.webank.wedatasphere.linkis.entrance.conf.EntranceConfiguration import com.webank.wedatasphere.linkis.entrance.interceptor.exception.VarSubstitutionException diff --git a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala index 59ea8a1ddca..d7fdf6b1213 100644 --- a/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala +++ b/ujes/entrance/src/main/scala/com/webank/wedatasphere/linkis/entrance/scheduler/EntranceGroupFactory.scala @@ -40,7 +40,7 @@ class EntranceGroupFactory extends GroupFactory { //TODO Query the database and get initCapacity, maxCapacity, maxRunningJobs, maxAskExecutorTimes(查询数据库，拿到initCapacity、maxCapacity、maxRunningJobs、maxAskExecutorTimes) val initCapacity = 100 val maxCapacity = 100 - var maxRunningJobs = 3 + var maxRunningJobs = EntranceConfiguration.WDS_LINKIS_INSTANCE.getValue val maxAskExecutorTimes = EntranceConfiguration.MAX_ASK_EXECUTOR_TIME.getValue.toLong if (groupName.split("_").length < 2){ logger.warn(s"name style of group: $groupName is not correct, we will set default value for the group") @@ -55,7 +55,7 @@ class EntranceGroupFactory extends GroupFactory { try{ maxRunningJobs = Integer.parseInt(keyAndValue.get(EntranceConfiguration.WDS_LINKIS_INSTANCE.key)) }catch{ - case t:Throwable => logger.error("get maxRunningJobs from configuration failed",t) + case t:Throwable => logger.warn("Get maxRunningJobs from configuration server failed! Next use the default value to continue.",t) } } logger.info("groupName: {} => maxRunningJobs is {}", groupName, maxRunningJobs) diff --git a/ujes/entranceclient/pom.xml b/ujes/entranceclient/pom.xml index 14745805d33..32f881269c2 100644 --- a/ujes/entranceclient/pom.xml +++ b/ujes/entranceclient/pom.xml @@ -23,7 +23,7 @@ linkis com.webank.wedatasphere.linkis - 0.5.0 + 0.8.0 linkis-ujes-entranceclient