2018-11-04 19:55:52,537 INFO [main] util.SignalUtils (Logging.scala:logInfo(54)) - Registered signal handler for TERM 2018-11-04 19:55:52,544 INFO [main] util.SignalUtils (Logging.scala:logInfo(54)) - Registered signal handler for HUP 2018-11-04 19:55:52,549 INFO [main] util.SignalUtils (Logging.scala:logInfo(54)) - Registered signal handler for INT 2018-11-04 19:55:55,057 INFO [main] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1173)) - mapred.output.compress is deprecated. Instead, use mapreduce.output.fileoutputformat.compress 2018-11-04 19:55:55,403 INFO [main] yarn.ApplicationMaster (Logging.scala:logInfo(54)) - Preparing Local resources 2018-11-04 19:55:58,670 INFO [main] yarn.ApplicationMaster (Logging.scala:logInfo(54)) - Prepared Local resources Map(__spark_libs__/guava-14.0.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/guava-14.0.1.jar" } size: 2189117 timestamp: 1541353858306 type: FILE visibility: PUBLIC, __spark_libs__/base64-2.3.8.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/base64-2.3.8.jar" } size: 17008 timestamp: 1541353852115 type: FILE visibility: PUBLIC, __spark_libs__/xbean-asm5-shaded-4.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/xbean-asm5-shaded-4.4.jar" } size: 144660 timestamp: 1541353877230 type: FILE visibility: PUBLIC, __spark_libs__/hive-beeline-1.2.1.spark2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hive-beeline-1.2.1.spark2.jar" } size: 138464 timestamp: 1541353860533 type: FILE visibility: PUBLIC, __spark_libs__/hive-cli-1.2.1.spark2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hive-cli-1.2.1.spark2.jar" } size: 40817 timestamp: 1541353860590 type: FILE visibility: PUBLIC, __spark_libs__/parquet-column-1.7.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/parquet-column-1.7.0.jar" } size: 917052 timestamp: 1541353869100 type: FILE visibility: PUBLIC, __spark_libs__/spark-core_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-core_2.11-2.0.0.jar" } size: 11689807 timestamp: 1541353873210 type: FILE visibility: PUBLIC, __spark_libs__/commons-codec-1.10.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-codec-1.10.jar" } size: 284184 timestamp: 1541353854166 type: FILE visibility: PUBLIC, __spark_libs__/spire_2.11-0.7.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spire_2.11-0.7.4.jar" } size: 7276083 timestamp: 1541353876428 type: FILE visibility: PUBLIC, __spark_libs__/datanucleus-rdbms-3.2.9.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/datanucleus-rdbms-3.2.9.jar" } size: 1809447 timestamp: 1541353856888 type: FILE visibility: PUBLIC, __spark_libs__/javolution-5.5.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/javolution-5.5.1.jar" } size: 395195 timestamp: 1541353863395 type: FILE visibility: PUBLIC, __spark_libs__/parquet-format-2.3.0-incubating.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/parquet-format-2.3.0-incubating.jar" } size: 387188 timestamp: 1541353869217 type: FILE visibility: PUBLIC, __spark_libs__/ivy-2.4.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/ivy-2.4.0.jar" } size: 1282424 timestamp: 1541353862156 type: FILE visibility: PUBLIC, __spark_libs__/jersey-client-2.22.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jersey-client-2.22.2.jar" } size: 167421 timestamp: 1541353863977 type: FILE visibility: PUBLIC, __spark_libs__/pyrolite-4.9.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/pyrolite-4.9.jar" } size: 93407 timestamp: 1541353869965 type: FILE visibility: PUBLIC, __spark_libs__/jetty-util-6.1.26.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jetty-util-6.1.26.jar" } size: 177131 timestamp: 1541353864746 type: FILE visibility: PUBLIC, __spark_libs__/scala-xml_2.11-1.0.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/scala-xml_2.11-1.0.2.jar" } size: 648678 timestamp: 1541353871587 type: FILE visibility: PUBLIC, __app__.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/tmp/radoop/training-vm/radoop-spark20-9.0.1.jar" } size: 175924 timestamp: 1540845237676 type: FILE visibility: PUBLIC, __spark_libs__/commons-compiler-2.7.6.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-compiler-2.7.6.jar" } size: 30595 timestamp: 1541353854678 type: FILE visibility: PUBLIC, __spark_libs__/commons-io-2.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-io-2.4.jar" } size: 185140 timestamp: 1541353854947 type: FILE visibility: PUBLIC, __spark_libs__/objenesis-2.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/objenesis-2.1.jar" } size: 41755 timestamp: 1541353868899 type: FILE visibility: PUBLIC, __spark_libs__/jersey-media-jaxb-2.22.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jersey-media-jaxb-2.22.2.jar" } size: 72733 timestamp: 1541353864556 type: FILE visibility: PUBLIC, __spark_libs__/parquet-jackson-1.7.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/parquet-jackson-1.7.0.jar" } size: 1048110 timestamp: 1541353869803 type: FILE visibility: PUBLIC, __spark_libs__/mx4j-3.0.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/mx4j-3.0.2.jar" } size: 409467 timestamp: 1541353868334 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-annotations-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-annotations-2.7.2.jar" } size: 17385 timestamp: 1541353858431 type: FILE visibility: PUBLIC, __spark_libs__/javax.servlet-api-3.1.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/javax.servlet-api-3.1.0.jar" } size: 95806 timestamp: 1541353863286 type: FILE visibility: PUBLIC, __spark_libs__/spire-macros_2.11-0.7.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spire-macros_2.11-0.7.4.jar" } size: 86811 timestamp: 1541353876193 type: FILE visibility: PUBLIC, __spark_libs__/apacheds-kerberos-codec-2.0.0-M15.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/apacheds-kerberos-codec-2.0.0-M15.jar" } size: 691479 timestamp: 1541353851566 type: FILE visibility: PUBLIC, __spark_libs__/scala-parser-combinators_2.11-1.0.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/scala-parser-combinators_2.11-1.0.4.jar" } size: 423753 timestamp: 1541353871041 type: FILE visibility: PUBLIC, __spark_libs__/commons-logging-1.1.3.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-logging-1.1.3.jar" } size: 62050 timestamp: 1541353855107 type: FILE visibility: PUBLIC, __spark_libs__/calcite-avatica-1.2.0-incubating.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/calcite-avatica-1.2.0-incubating.jar" } size: 258370 timestamp: 1541353853265 type: FILE visibility: PUBLIC, __spark_libs__/scala-library-2.11.8.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/scala-library-2.11.8.jar" } size: 5744974 timestamp: 1541353871017 type: FILE visibility: PUBLIC, __spark_libs__/stream-2.7.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/stream-2.7.0.jar" } size: 174351 timestamp: 1541353877017 type: FILE visibility: PUBLIC, __spark_libs__/javax.annotation-api-1.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/javax.annotation-api-1.2.jar" } size: 26366 timestamp: 1541353863158 type: FILE visibility: PUBLIC, __spark_libs__/parquet-common-1.7.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/parquet-common-1.7.0.jar" } size: 21575 timestamp: 1541353869130 type: FILE visibility: PUBLIC, __spark_libs__/opencsv-2.3.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/opencsv-2.3.jar" } size: 19827 timestamp: 1541353868923 type: FILE visibility: PUBLIC, __spark_libs__/commons-lang-2.6.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-lang-2.6.jar" } size: 284220 timestamp: 1541353854975 type: FILE visibility: PUBLIC, __spark_libs__/guice-servlet-3.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/guice-servlet-3.0.jar" } size: 65012 timestamp: 1541353858384 type: FILE visibility: PUBLIC, __spark_libs__/metrics-graphite-3.1.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/metrics-graphite-3.1.2.jar" } size: 20852 timestamp: 1541353868145 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-yarn-server-web-proxy-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-yarn-server-web-proxy-2.7.2.jar" } size: 34953 timestamp: 1541353860487 type: FILE visibility: PUBLIC, __spark_libs__/commons-math3-3.4.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-math3-3.4.1.jar" } size: 2035066 timestamp: 1541353855577 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-auth-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-auth-2.7.2.jar" } size: 70685 timestamp: 1541353858468 type: FILE visibility: PUBLIC, __spark_libs__/hk2-locator-2.4.0-b34.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hk2-locator-2.4.0-b34.jar" } size: 181271 timestamp: 1541353861796 type: FILE visibility: PUBLIC, __spark_libs__/commons-collections-3.2.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-collections-3.2.2.jar" } size: 588337 timestamp: 1541353854210 type: FILE visibility: PUBLIC, __spark_libs__/kryo-shaded-3.0.3.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/kryo-shaded-3.0.3.jar" } size: 358390 timestamp: 1541353866155 type: FILE visibility: PUBLIC, __spark_libs__/arpack_combined_all-0.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/arpack_combined_all-0.1.jar" } size: 1194003 timestamp: 1541353851842 type: FILE visibility: PUBLIC, __spark_libs__/spark-catalyst_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-catalyst_2.11-2.0.0.jar" } size: 6768881 timestamp: 1541353872670 type: FILE visibility: PUBLIC, __spark_libs__/jackson-module-paranamer-2.6.5.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jackson-module-paranamer-2.6.5.jar" } size: 41263 timestamp: 1541353862901 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-mapreduce-client-jobclient-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-mapreduce-client-jobclient-2.7.2.jar" } size: 38213 timestamp: 1541353859699 type: FILE visibility: PUBLIC, __spark_libs__/curator-client-2.6.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/curator-client-2.6.0.jar" } size: 68866 timestamp: 1541353856177 type: FILE visibility: PUBLIC, __spark_libs__/spark-tags_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-tags_2.11-2.0.0.jar" } size: 15303 timestamp: 1541353875957 type: FILE visibility: PUBLIC, __spark_libs__/lz4-1.3.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/lz4-1.3.0.jar" } size: 236880 timestamp: 1541353867159 type: FILE visibility: PUBLIC, __spark_libs__/RoaringBitmap-0.5.11.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/RoaringBitmap-0.5.11.jar" } size: 201928 timestamp: 1541353850708 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-client-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-client-2.7.2.jar" } size: 2545 timestamp: 1541353858507 type: FILE visibility: PUBLIC, __spark_libs__/javax.inject-1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/javax.inject-1.jar" } size: 2497 timestamp: 1541353863193 type: FILE visibility: PUBLIC, __spark_libs__/spark-unsafe_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-unsafe_2.11-2.0.0.jar" } size: 41081 timestamp: 1541353876006 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-yarn-common-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-yarn-common-2.7.2.jar" } size: 1653294 timestamp: 1541353860382 type: FILE visibility: PUBLIC, __spark_libs__/datanucleus-core-3.2.10.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/datanucleus-core-3.2.10.jar" } size: 1890075 timestamp: 1541353856774 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-yarn-server-common-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-yarn-server-common-2.7.2.jar" } size: 364376 timestamp: 1541353860435 type: FILE visibility: PUBLIC, __spark_libs__/commons-lang3-3.3.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-lang3-3.3.2.jar" } size: 412739 timestamp: 1541353855055 type: FILE visibility: PUBLIC, __spark_libs__/parquet-hadoop-bundle-1.6.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/parquet-hadoop-bundle-1.6.0.jar" } size: 2796935 timestamp: 1541353869775 type: FILE visibility: PUBLIC, __spark_libs__/javax.ws.rs-api-2.0.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/javax.ws.rs-api-2.0.1.jar" } size: 115534 timestamp: 1541353863336 type: FILE visibility: PUBLIC, __spark_libs__/validation-api-1.1.0.Final.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/validation-api-1.1.0.Final.jar" } size: 63777 timestamp: 1541353877186 type: FILE visibility: PUBLIC, __spark_libs__/jersey-container-servlet-2.22.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jersey-container-servlet-2.22.2.jar" } size: 18098 timestamp: 1541353864437 type: FILE visibility: PUBLIC, __spark_libs__/parquet-encoding-1.7.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/parquet-encoding-1.7.0.jar" } size: 285447 timestamp: 1541353869176 type: FILE visibility: PUBLIC, __spark_libs__/jpam-1.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jpam-1.1.jar" } size: 12131 timestamp: 1541353865331 type: FILE visibility: PUBLIC, __spark_libs__/oro-2.0.8.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/oro-2.0.8.jar" } size: 65261 timestamp: 1541353868974 type: FILE visibility: PUBLIC, __spark_libs__/jackson-databind-2.6.5.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jackson-databind-2.6.5.jar" } size: 1171380 timestamp: 1541353862381 type: FILE visibility: PUBLIC, __spark_libs__/parquet-hadoop-1.7.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/parquet-hadoop-1.7.0.jar" } size: 209622 timestamp: 1541353869308 type: FILE visibility: PUBLIC, __spark_libs__/java-xmlbuilder-1.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/java-xmlbuilder-1.0.jar" } size: 10023 timestamp: 1541353863056 type: FILE visibility: PUBLIC, __spark_libs__/netty-3.8.0.Final.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/netty-3.8.0.Final.jar" } size: 1230201 timestamp: 1541353868392 type: FILE visibility: PUBLIC, __spark_libs__/snappy-0.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/snappy-0.2.jar" } size: 48720 timestamp: 1541353872115 type: FILE visibility: PUBLIC, __spark_libs__/ST4-4.0.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/ST4-4.0.4.jar" } size: 236660 timestamp: 1541353850771 type: FILE visibility: PUBLIC, __spark_libs__/calcite-linq4j-1.2.0-incubating.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/calcite-linq4j-1.2.0-incubating.jar" } size: 442406 timestamp: 1541353853815 type: FILE visibility: PUBLIC, __spark_libs__/avro-1.7.7.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/avro-1.7.7.jar" } size: 436303 timestamp: 1541353851942 type: FILE visibility: PUBLIC, radoop-common-9.0.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/tmp/radoop/training-vm/radoop-common-9.0.1.jar" } size: 94743 timestamp: 1540845237588 type: FILE visibility: PUBLIC, __spark_libs__/apache-log4j-extras-1.2.17.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/apache-log4j-extras-1.2.17.jar" } size: 448794 timestamp: 1541353851412 type: FILE visibility: PUBLIC, __spark_libs__/metrics-json-3.1.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/metrics-json-3.1.2.jar" } size: 15827 timestamp: 1541353868169 type: FILE visibility: PUBLIC, __spark_libs__/spark-mllib-local_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-mllib-local_2.11-2.0.0.jar" } size: 176742 timestamp: 1541353874178 type: FILE visibility: PUBLIC, __spark_libs__/json4s-jackson_2.11-3.2.11.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/json4s-jackson_2.11-3.2.11.jar" } size: 40341 timestamp: 1541353865470 type: FILE visibility: PUBLIC, __spark_libs__/avro-ipc-1.7.7.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/avro-ipc-1.7.7.jar" } size: 192993 timestamp: 1541353851978 type: FILE visibility: PUBLIC, __spark_libs__/commons-pool-1.5.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-pool-1.5.4.jar" } size: 96221 timestamp: 1541353856041 type: FILE visibility: PUBLIC, __spark_libs__/hive-metastore-1.2.1.spark2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hive-metastore-1.2.1.spark2.jar" } size: 5505200 timestamp: 1541353861743 type: FILE visibility: PUBLIC, __spark_libs__/scala-reflect-2.11.8.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/scala-reflect-2.11.8.jar" } size: 4573750 timestamp: 1541353871547 type: FILE visibility: PUBLIC, __spark_libs__/super-csv-2.2.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/super-csv-2.2.0.jar" } size: 93210 timestamp: 1541353877098 type: FILE visibility: PUBLIC, __spark_libs__/protobuf-java-2.5.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/protobuf-java-2.5.0.jar" } size: 533455 timestamp: 1541353869901 type: FILE visibility: PUBLIC, __spark_libs__/spark-hive-thriftserver_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-hive-thriftserver_2.11-2.0.0.jar" } size: 1813845 timestamp: 1541353873690 type: FILE visibility: PUBLIC, __spark_libs__/antlr-2.7.7.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/antlr-2.7.7.jar" } size: 445288 timestamp: 1541353850966 type: FILE visibility: PUBLIC, __spark_libs__/curator-recipes-2.6.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/curator-recipes-2.6.0.jar" } size: 248171 timestamp: 1541353856651 type: FILE visibility: PUBLIC, __spark_libs__/jsp-api-2.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jsp-api-2.1.jar" } size: 100636 timestamp: 1541353865527 type: FILE visibility: PUBLIC, __spark_libs__/breeze_2.11-0.11.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/breeze_2.11-0.11.2.jar" } size: 13448966 timestamp: 1541353853217 type: FILE visibility: PUBLIC, __spark_libs__/commons-beanutils-core-1.8.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-beanutils-core-1.8.0.jar" } size: 206035 timestamp: 1541353854062 type: FILE visibility: PUBLIC, __spark_libs__/metrics-core-3.1.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/metrics-core-3.1.2.jar" } size: 112558 timestamp: 1541353867698 type: FILE visibility: PUBLIC, __spark_libs__/jackson-mapper-asl-1.9.13.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jackson-mapper-asl-1.9.13.jar" } size: 780664 timestamp: 1541353862472 type: FILE visibility: PUBLIC, __spark_libs__/antlr-runtime-3.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/antlr-runtime-3.4.jar" } size: 164368 timestamp: 1541353851066 type: FILE visibility: PUBLIC, __spark_libs__/jackson-annotations-2.6.5.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jackson-annotations-2.6.5.jar" } size: 46983 timestamp: 1541353862185 type: FILE visibility: PUBLIC, __spark_libs__/libthrift-0.9.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/libthrift-0.9.2.jar" } size: 227712 timestamp: 1541353867075 type: FILE visibility: PUBLIC, __spark_libs__/spark-hive_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-hive_2.11-2.0.0.jar" } size: 1033778 timestamp: 1541353874132 type: FILE visibility: PUBLIC, __spark_libs__/jackson-jaxrs-1.9.13.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jackson-jaxrs-1.9.13.jar" } size: 18336 timestamp: 1541353862429 type: FILE visibility: PUBLIC, __spark_libs__/jersey-guava-2.22.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jersey-guava-2.22.2.jar" } size: 971310 timestamp: 1541353864502 type: FILE visibility: PUBLIC, __spark_libs__/minlog-1.3.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/minlog-1.3.0.jar" } size: 5711 timestamp: 1541353868284 type: FILE visibility: PUBLIC, __spark_libs__/httpclient-4.5.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/httpclient-4.5.2.jar" } size: 736658 timestamp: 1541353862020 type: FILE visibility: PUBLIC, __spark_libs__/paranamer-2.3.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/paranamer-2.3.jar" } size: 29555 timestamp: 1541353869052 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-mapreduce-client-shuffle-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-mapreduce-client-shuffle-2.7.2.jar" } size: 48268 timestamp: 1541353859748 type: FILE visibility: PUBLIC, __spark_libs__/hive-exec-1.2.1.spark2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hive-exec-1.2.1.spark2.jar" } size: 11498852 timestamp: 1541353861194 type: FILE visibility: PUBLIC, __spark_libs__/libfb303-0.9.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/libfb303-0.9.2.jar" } size: 313686 timestamp: 1541353867043 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-mapreduce-client-common-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-mapreduce-client-common-2.7.2.jar" } size: 753123 timestamp: 1541353859601 type: FILE visibility: PUBLIC, __spark_libs__/spark-mllib_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-mllib_2.11-2.0.0.jar" } size: 5827607 timestamp: 1541353874631 type: FILE visibility: PUBLIC, __spark_libs__/parquet-generator-1.7.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/parquet-generator-1.7.0.jar" } size: 21243 timestamp: 1541353869249 type: FILE visibility: PUBLIC, __spark_libs__/jodd-core-3.5.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jodd-core-3.5.2.jar" } size: 427780 timestamp: 1541353865302 type: FILE visibility: PUBLIC, __spark_libs__/jackson-xc-1.9.13.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jackson-xc-1.9.13.jar" } size: 27084 timestamp: 1541353862971 type: FILE visibility: PUBLIC, __spark_libs__/javassist-3.18.1-GA.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/javassist-3.18.1-GA.jar" } size: 714194 timestamp: 1541353863121 type: FILE visibility: PUBLIC, __spark_libs__/jersey-common-2.22.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jersey-common-2.22.2.jar" } size: 698375 timestamp: 1541353864009 type: FILE visibility: PUBLIC, __spark_libs__/avro-mapred-1.7.7-hadoop2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/avro-mapred-1.7.7-hadoop2.jar" } size: 180736 timestamp: 1541353852055 type: FILE visibility: PUBLIC, __spark_libs__/spark-network-shuffle_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-network-shuffle_2.11-2.0.0.jar" } size: 58625 timestamp: 1541353874854 type: FILE visibility: PUBLIC, __spark_libs__/log4j-1.2.17.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/log4j-1.2.17.jar" } size: 489884 timestamp: 1541353867114 type: FILE visibility: PUBLIC, __spark_libs__/metrics-jvm-3.1.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/metrics-jvm-3.1.2.jar" } size: 39280 timestamp: 1541353868218 type: FILE visibility: PUBLIC, __spark_libs__/api-asn1-api-1.0.0-M20.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/api-asn1-api-1.0.0-M20.jar" } size: 16560 timestamp: 1541353851659 type: FILE visibility: PUBLIC, __spark_libs__/htrace-core-3.1.0-incubating.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/htrace-core-3.1.0-incubating.jar" } size: 1475955 timestamp: 1541353861947 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-yarn-client-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-yarn-client-2.7.2.jar" } size: 142639 timestamp: 1541353860297 type: FILE visibility: PUBLIC, __spark_libs__/aopalliance-repackaged-2.4.0-b34.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/aopalliance-repackaged-2.4.0-b34.jar" } size: 14766 timestamp: 1541353851318 type: FILE visibility: PUBLIC, __spark_conf__ -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/home/training-vm/.sparkStaging/application_1541356129398_0005/__spark_conf__.zip" } size: 29734 timestamp: 1541357747932 type: ARCHIVE visibility: PRIVATE, __spark_libs__/hadoop-hdfs-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-hdfs-2.7.2.jar" } size: 8268375 timestamp: 1541353859526 type: FILE visibility: PUBLIC, __spark_libs__/netty-all-4.0.29.Final.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/netty-all-4.0.29.Final.jar" } size: 2054931 timestamp: 1541353868458 type: FILE visibility: PUBLIC, __spark_libs__/scala-compiler-2.11.8.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/scala-compiler-2.11.8.jar" } size: 15487351 timestamp: 1541353870542 type: FILE visibility: PUBLIC, __spark_libs__/core-1.1.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/core-1.1.2.jar" } size: 164422 timestamp: 1541353856136 type: FILE visibility: PUBLIC, __spark_libs__/mail-1.4.7.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/mail-1.4.7.jar" } size: 521157 timestamp: 1541353867611 type: FILE visibility: PUBLIC, __spark_libs__/jul-to-slf4j-1.7.16.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jul-to-slf4j-1.7.16.jar" } size: 4596 timestamp: 1541353866082 type: FILE visibility: PUBLIC, __spark_libs__/snappy-java-1.1.2.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/snappy-java-1.1.2.4.jar" } size: 1056032 timestamp: 1541353872164 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-yarn-api-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-yarn-api-2.7.2.jar" } size: 2015575 timestamp: 1541353860256 type: FILE visibility: PUBLIC, __spark_libs__/xmlenc-0.52.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/xmlenc-0.52.jar" } size: 15010 timestamp: 1541353877306 type: FILE visibility: PUBLIC, __spark_libs__/spark-yarn_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-yarn_2.11-2.0.0.jar" } size: 663766 timestamp: 1541353876091 type: FILE visibility: PUBLIC, __spark_libs__/guice-3.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/guice-3.0.jar" } size: 710492 timestamp: 1541353858334 type: FILE visibility: PUBLIC, __spark_libs__/calcite-core-1.2.0-incubating.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/calcite-core-1.2.0-incubating.jar" } size: 3519262 timestamp: 1541353853769 type: FILE visibility: PUBLIC, __spark_libs__/xercesImpl-2.9.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/xercesImpl-2.9.1.jar" } size: 1229125 timestamp: 1541353877280 type: FILE visibility: PUBLIC, __spark_libs__/jets3t-0.9.3.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jets3t-0.9.3.jar" } size: 2041628 timestamp: 1541353864684 type: FILE visibility: PUBLIC, __spark_libs__/httpcore-4.4.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/httpcore-4.4.4.jar" } size: 326724 timestamp: 1541353862086 type: FILE visibility: PUBLIC, __spark_libs__/derby-10.11.1.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/derby-10.11.1.1.jar" } size: 3103132 timestamp: 1541353857400 type: FILE visibility: PUBLIC, __spark_libs__/json4s-ast_2.11-3.2.11.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/json4s-ast_2.11-3.2.11.jar" } size: 82421 timestamp: 1541353865393 type: FILE visibility: PUBLIC, __spark_libs__/slf4j-log4j12-1.7.16.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/slf4j-log4j12-1.7.16.jar" } size: 9939 timestamp: 1541353872084 type: FILE visibility: PUBLIC, __spark_libs__/jackson-core-2.6.5.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jackson-core-2.6.5.jar" } size: 258876 timestamp: 1541353862246 type: FILE visibility: PUBLIC, __spark_libs__/jaxb-api-2.2.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jaxb-api-2.2.2.jar" } size: 105134 timestamp: 1541353863855 type: FILE visibility: PUBLIC, __spark_libs__/pmml-model-1.2.15.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/pmml-model-1.2.15.jar" } size: 654216 timestamp: 1541353869835 type: FILE visibility: PUBLIC, __spark_libs__/jcl-over-slf4j-1.7.16.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jcl-over-slf4j-1.7.16.jar" } size: 16430 timestamp: 1541353863902 type: FILE visibility: PUBLIC, __spark_libs__/datanucleus-api-jdo-3.2.6.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/datanucleus-api-jdo-3.2.6.jar" } size: 339666 timestamp: 1541353856684 type: FILE visibility: PUBLIC, __spark_libs__/jdo-api-3.0.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jdo-api-3.0.1.jar" } size: 201124 timestamp: 1541353863943 type: FILE visibility: PUBLIC, __spark_libs__/curator-framework-2.6.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/curator-framework-2.6.0.jar" } size: 185245 timestamp: 1541353856210 type: FILE visibility: PUBLIC, __spark_libs__/jackson-module-scala_2.11-2.6.5.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jackson-module-scala_2.11-2.6.5.jar" } size: 515604 timestamp: 1541353862932 type: FILE visibility: PUBLIC, __spark_libs__/commons-beanutils-1.7.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-beanutils-1.7.0.jar" } size: 188671 timestamp: 1541353854008 type: FILE visibility: PUBLIC, __spark_libs__/jta-1.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jta-1.1.jar" } size: 15071 timestamp: 1541353865625 type: FILE visibility: PUBLIC, __spark_libs__/aopalliance-1.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/aopalliance-1.0.jar" } size: 4467 timestamp: 1541353851252 type: FILE visibility: PUBLIC, __spark_libs__/spark-sketch_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-sketch_2.11-2.0.0.jar" } size: 29540 timestamp: 1541353875316 type: FILE visibility: PUBLIC, __spark_libs__/bcprov-jdk15on-1.51.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/bcprov-jdk15on-1.51.jar" } size: 2842667 timestamp: 1541353852678 type: FILE visibility: PUBLIC, __spark_libs__/slf4j-api-1.7.16.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/slf4j-api-1.7.16.jar" } size: 40509 timestamp: 1541353871649 type: FILE visibility: PUBLIC, __spark_libs__/commons-dbcp-1.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-dbcp-1.4.jar" } size: 160519 timestamp: 1541353854828 type: FILE visibility: PUBLIC, __spark_libs__/commons-digester-1.8.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-digester-1.8.jar" } size: 143602 timestamp: 1541353854872 type: FILE visibility: PUBLIC, __spark_libs__/activation-1.1.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/activation-1.1.1.jar" } size: 69409 timestamp: 1541353850852 type: FILE visibility: PUBLIC, __spark_libs__/scalap-2.11.8.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/scalap-2.11.8.jar" } size: 802818 timestamp: 1541353871618 type: FILE visibility: PUBLIC, __spark_libs__/jersey-server-2.22.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jersey-server-2.22.2.jar" } size: 951701 timestamp: 1541353864620 type: FILE visibility: PUBLIC, __spark_libs__/janino-2.7.8.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/janino-2.7.8.jar" } size: 613299 timestamp: 1541353863015 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-mapreduce-client-app-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-mapreduce-client-app-2.7.2.jar" } size: 516614 timestamp: 1541353859564 type: FILE visibility: PUBLIC, __spark_libs__/jersey-container-servlet-core-2.22.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jersey-container-servlet-core-2.22.2.jar" } size: 66270 timestamp: 1541353864462 type: FILE visibility: PUBLIC, __spark_libs__/apacheds-i18n-2.0.0-M15.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/apacheds-i18n-2.0.0-M15.jar" } size: 44925 timestamp: 1541353851481 type: FILE visibility: PUBLIC, __spark_libs__/breeze-macros_2.11-0.11.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/breeze-macros_2.11-0.11.2.jar" } size: 135552 timestamp: 1541353852805 type: FILE visibility: PUBLIC, __spark_libs__/javax.inject-2.4.0-b34.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/javax.inject-2.4.0-b34.jar" } size: 5950 timestamp: 1541353863231 type: FILE visibility: PUBLIC, __spark_libs__/univocity-parsers-2.1.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/univocity-parsers-2.1.1.jar" } size: 290506 timestamp: 1541353877142 type: FILE visibility: PUBLIC, __spark_libs__/chill-java-0.8.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/chill-java-0.8.0.jar" } size: 50619 timestamp: 1541353853897 type: FILE visibility: PUBLIC, __spark_libs__/jetty-6.1.26.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jetty-6.1.26.jar" } size: 539912 timestamp: 1541353864719 type: FILE visibility: PUBLIC, __spark_libs__/osgi-resource-locator-1.0.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/osgi-resource-locator-1.0.1.jar" } size: 20235 timestamp: 1541353869010 type: FILE visibility: PUBLIC, __spark_libs__/commons-httpclient-3.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-httpclient-3.1.jar" } size: 305001 timestamp: 1541353854919 type: FILE visibility: PUBLIC, __spark_libs__/commons-compress-1.4.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-compress-1.4.1.jar" } size: 241367 timestamp: 1541353854730 type: FILE visibility: PUBLIC, __spark_libs__/compress-lzf-1.0.3.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/compress-lzf-1.0.3.jar" } size: 79845 timestamp: 1541353856075 type: FILE visibility: PUBLIC, __spark_libs__/gson-2.2.4.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/gson-2.2.4.jar" } size: 190432 timestamp: 1541353857863 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-mapreduce-client-core-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-mapreduce-client-core-2.7.2.jar" } size: 1531485 timestamp: 1541353859656 type: FILE visibility: PUBLIC, __spark_libs__/commons-net-2.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-net-2.2.jar" } size: 212453 timestamp: 1541353856009 type: FILE visibility: PUBLIC, __spark_libs__/spark-repl_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-repl_2.11-2.0.0.jar" } size: 63316 timestamp: 1541353874888 type: FILE visibility: PUBLIC, __spark_libs__/stax-api-1.0-2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/stax-api-1.0-2.jar" } size: 23346 timestamp: 1541353876538 type: FILE visibility: PUBLIC, __spark_libs__/mesos-0.21.1-shaded-protobuf.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/mesos-0.21.1-shaded-protobuf.jar" } size: 1277883 timestamp: 1541353867659 type: FILE visibility: PUBLIC, __spark_libs__/eigenbase-properties-1.1.5.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/eigenbase-properties-1.1.5.jar" } size: 18482 timestamp: 1541353857430 type: FILE visibility: PUBLIC, __spark_libs__/jtransforms-2.4.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jtransforms-2.4.0.jar" } size: 764569 timestamp: 1541353866059 type: FILE visibility: PUBLIC, __spark_libs__/spark-sql_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-sql_2.11-2.0.0.jar" } size: 6127242 timestamp: 1541353875795 type: FILE visibility: PUBLIC, __spark_libs__/chill_2.11-0.8.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/chill_2.11-0.8.0.jar" } size: 223573 timestamp: 1541353853947 type: FILE visibility: PUBLIC, __spark_libs__/api-util-1.0.0-M20.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/api-util-1.0.0-M20.jar" } size: 79912 timestamp: 1541353851738 type: FILE visibility: PUBLIC, __spark_libs__/hk2-api-2.4.0-b34.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hk2-api-2.4.0-b34.jar" } size: 178947 timestamp: 1541353861765 type: FILE visibility: PUBLIC, __spark_libs__/hk2-utils-2.4.0-b34.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hk2-utils-2.4.0-b34.jar" } size: 118973 timestamp: 1541353861868 type: FILE visibility: PUBLIC, __spark_libs__/stringtemplate-3.2.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/stringtemplate-3.2.1.jar" } size: 148627 timestamp: 1541353877054 type: FILE visibility: PUBLIC, __spark_libs__/antlr4-runtime-4.5.3.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/antlr4-runtime-4.5.3.jar" } size: 302248 timestamp: 1541353851179 type: FILE visibility: PUBLIC, __spark_libs__/json-20090211.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/json-20090211.jar" } size: 45944 timestamp: 1541353865360 type: FILE visibility: PUBLIC, __spark_libs__/stax-api-1.0.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/stax-api-1.0.1.jar" } size: 26514 timestamp: 1541353876991 type: FILE visibility: PUBLIC, __spark_libs__/spark-network-common_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-network-common_2.11-2.0.0.jar" } size: 2350593 timestamp: 1541353874786 type: FILE visibility: PUBLIC, __spark_libs__/JavaEWAH-0.3.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/JavaEWAH-0.3.2.jar" } size: 16993 timestamp: 1541353850622 type: FILE visibility: PUBLIC, __spark_libs__/spark-graphx_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-graphx_2.11-2.0.0.jar" } size: 680104 timestamp: 1541353873241 type: FILE visibility: PUBLIC, __spark_libs__/bonecp-0.8.0.RELEASE.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/bonecp-0.8.0.RELEASE.jar" } size: 110600 timestamp: 1541353852718 type: FILE visibility: PUBLIC, __spark_libs__/json4s-core_2.11-3.2.11.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/json4s-core_2.11-3.2.11.jar" } size: 589462 timestamp: 1541353865429 type: FILE visibility: PUBLIC, __spark_libs__/jackson-core-asl-1.9.13.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jackson-core-asl-1.9.13.jar" } size: 232248 timestamp: 1541353862308 type: FILE visibility: PUBLIC, __spark_libs__/zookeeper-3.4.6.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/zookeeper-3.4.6.jar" } size: 792964 timestamp: 1541353877397 type: FILE visibility: PUBLIC, __spark_libs__/pmml-schema-1.2.15.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/pmml-schema-1.2.15.jar" } size: 5310 timestamp: 1541353869869 type: FILE visibility: PUBLIC, __spark_libs__/xz-1.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/xz-1.0.jar" } size: 94672 timestamp: 1541353877353 type: FILE visibility: PUBLIC, __spark_libs__/commons-cli-1.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-cli-1.2.jar" } size: 41123 timestamp: 1541353854120 type: FILE visibility: PUBLIC, __spark_libs__/hadoop-common-2.7.2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hadoop-common-2.7.2.jar" } size: 3443040 timestamp: 1541353859023 type: FILE visibility: PUBLIC, __spark_libs__/py4j-0.10.1.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/py4j-0.10.1.jar" } size: 106731 timestamp: 1541353869928 type: FILE visibility: PUBLIC, __spark_libs__/joda-time-2.9.3.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/joda-time-2.9.3.jar" } size: 627814 timestamp: 1541353864853 type: FILE visibility: PUBLIC, __spark_libs__/spark-launcher_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-launcher_2.11-2.0.0.jar" } size: 65651 timestamp: 1541353874154 type: FILE visibility: PUBLIC, __spark_libs__/spark-streaming_2.11-2.0.0.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/spark-streaming_2.11-2.0.0.jar" } size: 2085415 timestamp: 1541353875920 type: FILE visibility: PUBLIC, __spark_libs__/jline-2.12.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jline-2.12.jar" } size: 213854 timestamp: 1541353864786 type: FILE visibility: PUBLIC, __spark_libs__/commons-configuration-1.6.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/commons-configuration-1.6.jar" } size: 298829 timestamp: 1541353854785 type: FILE visibility: PUBLIC, __spark_libs__/leveldbjni-all-1.8.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/leveldbjni-all-1.8.jar" } size: 1045744 timestamp: 1541353866613 type: FILE visibility: PUBLIC, __spark_libs__/hive-jdbc-1.2.1.spark2.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/hive-jdbc-1.2.1.spark2.jar" } size: 100680 timestamp: 1541353861219 type: FILE visibility: PUBLIC, __spark_libs__/jsr305-1.3.9.jar -> resource { scheme: "hdfs" host: "192.168.99.100" port: 9020 file: "/spark/jars/jsr305-1.3.9.jar" } size: 33015 timestamp: 1541353865579 type: FILE visibility: PUBLIC) 2018-11-04 19:55:58,768 INFO [main] yarn.ApplicationMaster (Logging.scala:logInfo(54)) - ApplicationAttemptId: appattempt_1541356129398_0005_000001 2018-11-04 19:55:58,785 INFO [main] spark.SecurityManager (Logging.scala:logInfo(54)) - Changing view acls to: training-vm 2018-11-04 19:55:58,789 INFO [main] spark.SecurityManager (Logging.scala:logInfo(54)) - Changing modify acls to: training-vm 2018-11-04 19:55:58,790 INFO [main] spark.SecurityManager (Logging.scala:logInfo(54)) - Changing view acls groups to: 2018-11-04 19:55:58,791 INFO [main] spark.SecurityManager (Logging.scala:logInfo(54)) - Changing modify acls groups to: 2018-11-04 19:55:58,793 INFO [main] spark.SecurityManager (Logging.scala:logInfo(54)) - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(training-vm); groups with view permissions: Set(); users with modify permissions: Set(training-vm); groups with modify permissions: Set() 2018-11-04 19:55:58,859 INFO [main] yarn.ApplicationMaster (Logging.scala:logInfo(54)) - Starting the user application in a separate Thread 2018-11-04 19:55:59,063 INFO [main] yarn.ApplicationMaster (Logging.scala:logInfo(54)) - Waiting for spark context initialization 2018-11-04 19:55:59,064 INFO [main] yarn.ApplicationMaster (Logging.scala:logInfo(54)) - Waiting for spark context initialization ... 2018-11-04 19:55:59,154 INFO [Driver] spark.SparkContext (Logging.scala:logInfo(54)) - Running Spark version 2.0.0 2018-11-04 19:55:59,370 INFO [Driver] spark.SecurityManager (Logging.scala:logInfo(54)) - Changing view acls to: training-vm 2018-11-04 19:55:59,384 INFO [Driver] spark.SecurityManager (Logging.scala:logInfo(54)) - Changing modify acls to: training-vm 2018-11-04 19:55:59,385 INFO [Driver] spark.SecurityManager (Logging.scala:logInfo(54)) - Changing view acls groups to: 2018-11-04 19:55:59,386 INFO [Driver] spark.SecurityManager (Logging.scala:logInfo(54)) - Changing modify acls groups to: 2018-11-04 19:55:59,386 INFO [Driver] spark.SecurityManager (Logging.scala:logInfo(54)) - SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(training-vm); groups with view permissions: Set(); users with modify permissions: Set(training-vm); groups with modify permissions: Set() 2018-11-04 19:56:00,638 INFO [Driver] util.Utils (Logging.scala:logInfo(54)) - Successfully started service 'sparkDriver' on port 36363. 2018-11-04 19:56:00,763 INFO [Driver] spark.SparkEnv (Logging.scala:logInfo(54)) - Registering MapOutputTracker 2018-11-04 19:56:00,854 INFO [Driver] spark.SparkEnv (Logging.scala:logInfo(54)) - Registering BlockManagerMaster 2018-11-04 19:56:00,927 INFO [Driver] storage.DiskBlockManager (Logging.scala:logInfo(54)) - Created local directory at /tmp/hadoop-training-vm/nm-local-dir/usercache/training-vm/appcache/application_1541356129398_0005/blockmgr-c331d696-f006-4f83-a2a1-6394e8de1f48 2018-11-04 19:56:01,002 INFO [Driver] memory.MemoryStore (Logging.scala:logInfo(54)) - MemoryStore started with capacity 413.9 MB 2018-11-04 19:56:01,392 INFO [Driver] spark.SparkEnv (Logging.scala:logInfo(54)) - Registering OutputCommitCoordinator 2018-11-04 19:56:01,856 INFO [Driver] util.log (Log.java:initialized(186)) - Logging initialized @11320ms 2018-11-04 19:56:02,091 INFO [Driver] ui.JettyUtils (Logging.scala:logInfo(54)) - Adding filter: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter 2018-11-04 19:56:02,363 INFO [Driver] server.Server (Server.java:doStart(327)) - jetty-9.2.z-SNAPSHOT 2018-11-04 19:56:02,466 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@797d62f4{/jobs,null,AVAILABLE} 2018-11-04 19:56:02,473 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@49b8a36e{/jobs/json,null,AVAILABLE} 2018-11-04 19:56:02,475 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@39e7e600{/jobs/job,null,AVAILABLE} 2018-11-04 19:56:02,476 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@6d99177{/jobs/job/json,null,AVAILABLE} 2018-11-04 19:56:02,478 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@14a1299a{/stages,null,AVAILABLE} 2018-11-04 19:56:02,479 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@188e4bb3{/stages/json,null,AVAILABLE} 2018-11-04 19:56:02,479 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@1e5f7944{/stages/stage,null,AVAILABLE} 2018-11-04 19:56:02,480 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@544701d1{/stages/stage/json,null,AVAILABLE} 2018-11-04 19:56:02,480 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@49371f3a{/stages/pool,null,AVAILABLE} 2018-11-04 19:56:02,481 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@238e067c{/stages/pool/json,null,AVAILABLE} 2018-11-04 19:56:02,488 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@46fede10{/storage,null,AVAILABLE} 2018-11-04 19:56:02,489 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@2adb1697{/storage/json,null,AVAILABLE} 2018-11-04 19:56:02,494 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@5b641239{/storage/rdd,null,AVAILABLE} 2018-11-04 19:56:02,495 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@80a42cf{/storage/rdd/json,null,AVAILABLE} 2018-11-04 19:56:02,503 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@38b40380{/environment,null,AVAILABLE} 2018-11-04 19:56:02,503 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@4a29c5af{/environment/json,null,AVAILABLE} 2018-11-04 19:56:02,505 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@31a7f309{/executors,null,AVAILABLE} 2018-11-04 19:56:02,509 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@6b61c3c8{/executors/json,null,AVAILABLE} 2018-11-04 19:56:02,511 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@73516527{/executors/threadDump,null,AVAILABLE} 2018-11-04 19:56:02,515 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@77d1c25f{/executors/threadDump/json,null,AVAILABLE} 2018-11-04 19:56:02,549 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@79604817{/static,null,AVAILABLE} 2018-11-04 19:56:02,550 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@1c7aab70{/,null,AVAILABLE} 2018-11-04 19:56:02,556 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@647a9d85{/api,null,AVAILABLE} 2018-11-04 19:56:02,557 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@1d40786f{/stages/stage/kill,null,AVAILABLE} 2018-11-04 19:56:02,590 INFO [Driver] server.ServerConnector (AbstractConnector.java:doStart(266)) - Started ServerConnector@721aa415{HTTP/1.1}{0.0.0.0:33605} 2018-11-04 19:56:02,590 INFO [Driver] server.Server (Server.java:doStart(379)) - Started @12069ms 2018-11-04 19:56:02,591 INFO [Driver] util.Utils (Logging.scala:logInfo(54)) - Successfully started service 'SparkUI' on port 33605. 2018-11-04 19:56:02,597 INFO [Driver] ui.SparkUI (Logging.scala:logInfo(54)) - Bound SparkUI to 0.0.0.0, and started at http://192.168.99.100:33605 2018-11-04 19:56:03,008 INFO [Driver] cluster.YarnClusterScheduler (Logging.scala:logInfo(54)) - Created YarnClusterScheduler 2018-11-04 19:56:03,149 INFO [Driver] cluster.SchedulerExtensionServices (Logging.scala:logInfo(54)) - Starting Yarn extension services with app application_1541356129398_0005 and attemptId Some(appattempt_1541356129398_0005_000001) 2018-11-04 19:56:03,225 INFO [Driver] util.Utils (Logging.scala:logInfo(54)) - Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 38651. 2018-11-04 19:56:03,230 INFO [Driver] netty.NettyBlockTransferService (Logging.scala:logInfo(54)) - Server created on 192.168.99.100:38651 2018-11-04 19:56:03,247 INFO [Driver] storage.BlockManagerMaster (Logging.scala:logInfo(54)) - Registering BlockManager BlockManagerId(driver, 192.168.99.100, 38651) 2018-11-04 19:56:03,265 INFO [dispatcher-event-loop-0] storage.BlockManagerMasterEndpoint (Logging.scala:logInfo(54)) - Registering block manager 192.168.99.100:38651 with 413.9 MB RAM, BlockManagerId(driver, 192.168.99.100, 38651) 2018-11-04 19:56:03,299 INFO [Driver] storage.BlockManagerMaster (Logging.scala:logInfo(54)) - Registered BlockManager BlockManagerId(driver, 192.168.99.100, 38651) 2018-11-04 19:56:04,060 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStart(744)) - Started o.s.j.s.ServletContextHandler@730db8c7{/metrics/json,null,AVAILABLE} 2018-11-04 19:56:04,403 INFO [dispatcher-event-loop-0] cluster.YarnSchedulerBackend$YarnSchedulerEndpoint (Logging.scala:logInfo(54)) - ApplicationMaster registered as NettyRpcEndpointRef(spark://YarnAM@192.168.99.100:36363) 2018-11-04 19:56:04,507 INFO [main] client.RMProxy (RMProxy.java:createRMProxy(98)) - Connecting to ResourceManager at /0.0.0.0:8030 2018-11-04 19:56:04,609 INFO [main] yarn.YarnRMClient (Logging.scala:logInfo(54)) - Registering the ApplicationMaster 2018-11-04 19:56:04,995 INFO [main] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Will request 2 executor containers, each with 1 cores and 1408 MB memory including 384 MB overhead 2018-11-04 19:56:05,006 INFO [main] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Canceled 0 container requests (locality no longer needed) 2018-11-04 19:56:05,061 INFO [main] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Submitted container request (host: Any, capability: ) 2018-11-04 19:56:05,062 INFO [main] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Submitted container request (host: Any, capability: ) 2018-11-04 19:56:05,204 INFO [main] yarn.ApplicationMaster (Logging.scala:logInfo(54)) - Started progress reporter thread with (heartbeat : 3000, initial allocation : 200) intervals 2018-11-04 19:56:06,666 INFO [Reporter] impl.AMRMClientImpl (AMRMClientImpl.java:populateNMTokens(360)) - Received new token for : trainingvm:36539 2018-11-04 19:56:06,687 INFO [Reporter] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Launching container container_1541356129398_0005_01_000002 for on host trainingvm 2018-11-04 19:56:06,690 INFO [Reporter] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Launching ExecutorRunnable. driverUrl: spark://CoarseGrainedScheduler@192.168.99.100:36363, executorHostname: trainingvm 2018-11-04 19:56:06,700 INFO [Reporter] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Received 1 containers from YARN, launching executors on 1 of them. 2018-11-04 19:56:06,711 INFO [ContainerLauncher-0] yarn.ExecutorRunnable (Logging.scala:logInfo(54)) - Starting Executor Container 2018-11-04 19:56:06,719 INFO [ContainerLauncher-0] impl.ContainerManagementProtocolProxy (ContainerManagementProtocolProxy.java:(81)) - yarn.client.max-cached-nodemanagers-proxies : 0 2018-11-04 19:56:06,723 INFO [ContainerLauncher-0] yarn.ExecutorRunnable (Logging.scala:logInfo(54)) - Setting up ContainerLaunchContext 2018-11-04 19:56:06,875 INFO [ContainerLauncher-0] yarn.ExecutorRunnable (Logging.scala:logInfo(54)) - =============================================================================== YARN executor launch context: env: CLASSPATH -> {{PWD}}{{PWD}}/__spark_conf__{{PWD}}/__spark_libs__/*$HADOOP_CONF_DIR$HADOOP_COMMON_HOME/share/hadoop/common/*$HADOOP_COMMON_HOME/share/hadoop/common/lib/*$HADOOP_HDFS_HOME/share/hadoop/hdfs/*$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*$HADOOP_YARN_HOME/share/hadoop/yarn/*$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/* SPARK_LOG_URL_STDERR -> http://trainingvm:8042/node/containerlogs/container_1541356129398_0005_01_000002/training-vm/stderr?start=-4096 SPARK_YARN_STAGING_DIR -> hdfs://192.168.99.100:9020/home/training-vm/.sparkStaging/application_1541356129398_0005 SPARK_USER -> training-vm SPARK_YARN_MODE -> true SPARK_LOG_URL_STDOUT -> http://trainingvm:8042/node/containerlogs/container_1541356129398_0005_01_000002/training-vm/stdout?start=-4096 command: {{JAVA_HOME}}/bin/java -server -Xmx1024m '-XX:MaxPermSize=128m' -Djava.io.tmpdir={{PWD}}/tmp '-Dspark.driver.port=36363' '-Dspark.ui.port=0' -Dspark.yarn.app.container.log.dir= -XX:OnOutOfMemoryError='kill %p' org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url spark://CoarseGrainedScheduler@192.168.99.100:36363 --executor-id 1 --hostname trainingvm --cores 1 --app-id application_1541356129398_0005 --user-class-path file:$PWD/__app__.jar --user-class-path file:$PWD/radoop-common-9.0.1.jar 1> /stdout 2> /stderr =============================================================================== 2018-11-04 19:56:06,886 INFO [ContainerLauncher-0] impl.ContainerManagementProtocolProxy (ContainerManagementProtocolProxy.java:newProxy(260)) - Opening proxy : trainingvm:36539 2018-11-04 19:56:08,316 INFO [Reporter] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Launching container container_1541356129398_0005_01_000003 for on host trainingvm 2018-11-04 19:56:08,317 INFO [Reporter] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Launching ExecutorRunnable. driverUrl: spark://CoarseGrainedScheduler@192.168.99.100:36363, executorHostname: trainingvm 2018-11-04 19:56:08,317 INFO [Reporter] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Received 1 containers from YARN, launching executors on 1 of them. 2018-11-04 19:56:08,340 INFO [ContainerLauncher-1] yarn.ExecutorRunnable (Logging.scala:logInfo(54)) - Starting Executor Container 2018-11-04 19:56:08,349 INFO [ContainerLauncher-1] impl.ContainerManagementProtocolProxy (ContainerManagementProtocolProxy.java:(81)) - yarn.client.max-cached-nodemanagers-proxies : 0 2018-11-04 19:56:08,350 INFO [ContainerLauncher-1] yarn.ExecutorRunnable (Logging.scala:logInfo(54)) - Setting up ContainerLaunchContext 2018-11-04 19:56:08,368 INFO [ContainerLauncher-1] yarn.ExecutorRunnable (Logging.scala:logInfo(54)) - =============================================================================== YARN executor launch context: env: CLASSPATH -> {{PWD}}{{PWD}}/__spark_conf__{{PWD}}/__spark_libs__/*$HADOOP_CONF_DIR$HADOOP_COMMON_HOME/share/hadoop/common/*$HADOOP_COMMON_HOME/share/hadoop/common/lib/*$HADOOP_HDFS_HOME/share/hadoop/hdfs/*$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*$HADOOP_YARN_HOME/share/hadoop/yarn/*$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/* SPARK_LOG_URL_STDERR -> http://trainingvm:8042/node/containerlogs/container_1541356129398_0005_01_000003/training-vm/stderr?start=-4096 SPARK_YARN_STAGING_DIR -> hdfs://192.168.99.100:9020/home/training-vm/.sparkStaging/application_1541356129398_0005 SPARK_USER -> training-vm SPARK_YARN_MODE -> true SPARK_LOG_URL_STDOUT -> http://trainingvm:8042/node/containerlogs/container_1541356129398_0005_01_000003/training-vm/stdout?start=-4096 command: {{JAVA_HOME}}/bin/java -server -Xmx1024m '-XX:MaxPermSize=128m' -Djava.io.tmpdir={{PWD}}/tmp '-Dspark.driver.port=36363' '-Dspark.ui.port=0' -Dspark.yarn.app.container.log.dir= -XX:OnOutOfMemoryError='kill %p' org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url spark://CoarseGrainedScheduler@192.168.99.100:36363 --executor-id 2 --hostname trainingvm --cores 1 --app-id application_1541356129398_0005 --user-class-path file:$PWD/__app__.jar --user-class-path file:$PWD/radoop-common-9.0.1.jar 1> /stdout 2> /stderr =============================================================================== 2018-11-04 19:56:08,375 INFO [ContainerLauncher-1] impl.ContainerManagementProtocolProxy (ContainerManagementProtocolProxy.java:newProxy(260)) - Opening proxy : trainingvm:36539 2018-11-04 19:56:11,395 INFO [Reporter] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Received 1 containers from YARN, launching executors on 0 of them. 2018-11-04 19:56:25,807 INFO [dispatcher-event-loop-0] cluster.YarnSchedulerBackend$YarnDriverEndpoint (Logging.scala:logInfo(54)) - Registered executor NettyRpcEndpointRef(null) (192.168.99.100:51932) with ID 1 2018-11-04 19:56:26,217 INFO [dispatcher-event-loop-0] storage.BlockManagerMasterEndpoint (Logging.scala:logInfo(54)) - Registering block manager trainingvm:36735 with 413.9 MB RAM, BlockManagerId(1, trainingvm, 36735) 2018-11-04 19:56:27,064 INFO [dispatcher-event-loop-0] cluster.YarnSchedulerBackend$YarnDriverEndpoint (Logging.scala:logInfo(54)) - Registered executor NettyRpcEndpointRef(null) (192.168.99.100:51936) with ID 2 2018-11-04 19:56:27,156 INFO [Driver] cluster.YarnClusterSchedulerBackend (Logging.scala:logInfo(54)) - SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.8 2018-11-04 19:56:27,157 INFO [Driver] cluster.YarnClusterScheduler (Logging.scala:logInfo(54)) - YarnClusterScheduler.postStartHook done 2018-11-04 19:56:27,405 INFO [dispatcher-event-loop-1] storage.BlockManagerMasterEndpoint (Logging.scala:logInfo(54)) - Registering block manager trainingvm:40527 with 413.9 MB RAM, BlockManagerId(2, trainingvm, 40527) commonPto: INPUT_DIR (String) : /tmp/radoop/training-vm/tmp_1541357744748_x0migqc/ EXAMPLE_SET_OUTPUT_DIR (String) : /tmp/radoop/training-vm/tmp_1541357744823_gsiifdt/ 2018-11-04 19:56:28,675 INFO [Driver] memory.MemoryStore (Logging.scala:logInfo(54)) - Block broadcast_0 stored as values in memory (estimated size 116.4 KB, free 413.8 MB) 2018-11-04 19:56:29,071 INFO [Driver] memory.MemoryStore (Logging.scala:logInfo(54)) - Block broadcast_0_piece0 stored as bytes in memory (estimated size 22.7 KB, free 413.8 MB) 2018-11-04 19:56:29,085 INFO [dispatcher-event-loop-0] storage.BlockManagerInfo (Logging.scala:logInfo(54)) - Added broadcast_0_piece0 in memory on 192.168.99.100:38651 (size: 22.7 KB, free: 413.9 MB) 2018-11-04 19:56:29,148 INFO [Driver] spark.SparkContext (Logging.scala:logInfo(54)) - Created broadcast 0 from textFile at SparkTestCountJobRunner.java:41 2018-11-04 19:56:29,721 INFO [Driver] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1173)) - mapred.tip.id is deprecated. Instead, use mapreduce.task.id 2018-11-04 19:56:29,722 INFO [Driver] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1173)) - mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id 2018-11-04 19:56:29,722 INFO [Driver] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1173)) - mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap 2018-11-04 19:56:29,722 INFO [Driver] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1173)) - mapred.task.partition is deprecated. Instead, use mapreduce.task.partition 2018-11-04 19:56:29,727 INFO [Driver] Configuration.deprecation (Configuration.java:warnOnceIfDeprecated(1173)) - mapred.job.id is deprecated. Instead, use mapreduce.job.id 2018-11-04 19:56:29,736 INFO [Driver] output.FileOutputCommitter (FileOutputCommitter.java:(100)) - File Output Committer Algorithm version is 1 2018-11-04 19:56:30,273 INFO [dispatcher-event-loop-1] storage.BlockManagerInfo (Logging.scala:logInfo(54)) - Removed broadcast_0_piece0 on 192.168.99.100:38651 in memory (size: 22.7 KB, free: 413.9 MB) 2018-11-04 19:56:30,598 INFO [Driver] spark.SparkContext (Logging.scala:logInfo(54)) - Starting job: saveAsTextFile at SparkTestCountJobRunner.java:53 2018-11-04 19:56:30,768 INFO [dag-scheduler-event-loop] scheduler.DAGScheduler (Logging.scala:logInfo(54)) - Got job 0 (saveAsTextFile at SparkTestCountJobRunner.java:53) with 1 output partitions 2018-11-04 19:56:30,770 INFO [dag-scheduler-event-loop] scheduler.DAGScheduler (Logging.scala:logInfo(54)) - Final stage: ResultStage 0 (saveAsTextFile at SparkTestCountJobRunner.java:53) 2018-11-04 19:56:30,772 INFO [dag-scheduler-event-loop] scheduler.DAGScheduler (Logging.scala:logInfo(54)) - Parents of final stage: List() 2018-11-04 19:56:30,774 INFO [dag-scheduler-event-loop] scheduler.DAGScheduler (Logging.scala:logInfo(54)) - Missing parents: List() 2018-11-04 19:56:30,829 INFO [dag-scheduler-event-loop] scheduler.DAGScheduler (Logging.scala:logInfo(54)) - Submitting ResultStage 0 (MapPartitionsRDD[4] at saveAsTextFile at SparkTestCountJobRunner.java:53), which has no missing parents 2018-11-04 19:56:31,153 INFO [dag-scheduler-event-loop] memory.MemoryStore (Logging.scala:logInfo(54)) - Block broadcast_1 stored as values in memory (estimated size 71.2 KB, free 413.9 MB) 2018-11-04 19:56:31,171 INFO [dag-scheduler-event-loop] memory.MemoryStore (Logging.scala:logInfo(54)) - Block broadcast_1_piece0 stored as bytes in memory (estimated size 25.5 KB, free 413.8 MB) 2018-11-04 19:56:31,173 INFO [dispatcher-event-loop-1] storage.BlockManagerInfo (Logging.scala:logInfo(54)) - Added broadcast_1_piece0 in memory on 192.168.99.100:38651 (size: 25.5 KB, free: 413.9 MB) 2018-11-04 19:56:31,180 INFO [dag-scheduler-event-loop] spark.SparkContext (Logging.scala:logInfo(54)) - Created broadcast 1 from broadcast at DAGScheduler.scala:1012 2018-11-04 19:56:31,208 INFO [dag-scheduler-event-loop] scheduler.DAGScheduler (Logging.scala:logInfo(54)) - Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[4] at saveAsTextFile at SparkTestCountJobRunner.java:53) 2018-11-04 19:56:31,217 INFO [dag-scheduler-event-loop] cluster.YarnClusterScheduler (Logging.scala:logInfo(54)) - Adding task set 0.0 with 1 tasks 2018-11-04 19:56:31,668 INFO [dispatcher-event-loop-0] scheduler.TaskSetManager (Logging.scala:logInfo(54)) - Starting task 0.0 in stage 0.0 (TID 0, trainingvm, partition 0, PROCESS_LOCAL, 5701 bytes) 2018-11-04 19:56:31,793 INFO [dispatcher-event-loop-0] cluster.YarnSchedulerBackend$YarnDriverEndpoint (Logging.scala:logInfo(54)) - Launching task 0 on executor id: 2 hostname: trainingvm. 2018-11-04 19:56:33,358 INFO [dispatcher-event-loop-1] storage.BlockManagerInfo (Logging.scala:logInfo(54)) - Added broadcast_1_piece0 in memory on trainingvm:40527 (size: 25.5 KB, free: 413.9 MB) 2018-11-04 19:56:34,959 INFO [task-result-getter-0] scheduler.TaskSetManager (Logging.scala:logInfo(54)) - Finished task 0.0 in stage 0.0 (TID 0) in 3683 ms on trainingvm (1/1) 2018-11-04 19:56:35,004 INFO [task-result-getter-0] cluster.YarnClusterScheduler (Logging.scala:logInfo(54)) - Removed TaskSet 0.0, whose tasks have all completed, from pool 2018-11-04 19:56:35,017 INFO [dag-scheduler-event-loop] scheduler.DAGScheduler (Logging.scala:logInfo(54)) - ResultStage 0 (saveAsTextFile at SparkTestCountJobRunner.java:53) finished in 3.747 s 2018-11-04 19:56:35,079 INFO [Driver] scheduler.DAGScheduler (Logging.scala:logInfo(54)) - Job 0 finished: saveAsTextFile at SparkTestCountJobRunner.java:53, took 4.480418 s 2018-11-04 19:56:35,323 INFO [Driver] server.ServerConnector (AbstractConnector.java:doStop(306)) - Stopped ServerConnector@721aa415{HTTP/1.1}{0.0.0.0:0} 2018-11-04 19:56:35,327 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@1d40786f{/stages/stage/kill,null,UNAVAILABLE} 2018-11-04 19:56:35,327 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@647a9d85{/api,null,UNAVAILABLE} 2018-11-04 19:56:35,327 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@1c7aab70{/,null,UNAVAILABLE} 2018-11-04 19:56:35,335 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@79604817{/static,null,UNAVAILABLE} 2018-11-04 19:56:35,336 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@77d1c25f{/executors/threadDump/json,null,UNAVAILABLE} 2018-11-04 19:56:35,337 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@73516527{/executors/threadDump,null,UNAVAILABLE} 2018-11-04 19:56:35,338 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@6b61c3c8{/executors/json,null,UNAVAILABLE} 2018-11-04 19:56:35,341 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@31a7f309{/executors,null,UNAVAILABLE} 2018-11-04 19:56:35,348 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@4a29c5af{/environment/json,null,UNAVAILABLE} 2018-11-04 19:56:35,351 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@38b40380{/environment,null,UNAVAILABLE} 2018-11-04 19:56:35,352 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@80a42cf{/storage/rdd/json,null,UNAVAILABLE} 2018-11-04 19:56:35,352 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@5b641239{/storage/rdd,null,UNAVAILABLE} 2018-11-04 19:56:35,352 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@2adb1697{/storage/json,null,UNAVAILABLE} 2018-11-04 19:56:35,352 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@46fede10{/storage,null,UNAVAILABLE} 2018-11-04 19:56:35,353 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@238e067c{/stages/pool/json,null,UNAVAILABLE} 2018-11-04 19:56:35,353 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@49371f3a{/stages/pool,null,UNAVAILABLE} 2018-11-04 19:56:35,353 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@544701d1{/stages/stage/json,null,UNAVAILABLE} 2018-11-04 19:56:35,353 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@1e5f7944{/stages/stage,null,UNAVAILABLE} 2018-11-04 19:56:35,354 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@188e4bb3{/stages/json,null,UNAVAILABLE} 2018-11-04 19:56:35,354 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@14a1299a{/stages,null,UNAVAILABLE} 2018-11-04 19:56:35,354 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@6d99177{/jobs/job/json,null,UNAVAILABLE} 2018-11-04 19:56:35,354 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@39e7e600{/jobs/job,null,UNAVAILABLE} 2018-11-04 19:56:35,355 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@49b8a36e{/jobs/json,null,UNAVAILABLE} 2018-11-04 19:56:35,355 INFO [Driver] handler.ContextHandler (ContextHandler.java:doStop(865)) - Stopped o.s.j.s.ServletContextHandler@797d62f4{/jobs,null,UNAVAILABLE} 2018-11-04 19:56:35,371 INFO [Driver] ui.SparkUI (Logging.scala:logInfo(54)) - Stopped Spark web UI at http://192.168.99.100:33605 2018-11-04 19:56:35,406 INFO [dispatcher-event-loop-1] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Driver requested a total number of 0 executor(s). 2018-11-04 19:56:35,407 INFO [Reporter] yarn.YarnAllocator (Logging.scala:logInfo(54)) - Canceling requests for 0 executor container(s) to have a new desired total 0 executors. 2018-11-04 19:56:35,407 WARN [Reporter] yarn.YarnAllocator (Logging.scala:logWarning(66)) - Expected to find pending requests, but found none. 2018-11-04 19:56:35,414 INFO [Driver] cluster.YarnClusterSchedulerBackend (Logging.scala:logInfo(54)) - Shutting down all executors 2018-11-04 19:56:35,422 INFO [dispatcher-event-loop-0] cluster.YarnSchedulerBackend$YarnDriverEndpoint (Logging.scala:logInfo(54)) - Asking each executor to shut down 2018-11-04 19:56:35,451 INFO [Driver] cluster.SchedulerExtensionServices (Logging.scala:logInfo(54)) - Stopping SchedulerExtensionServices (serviceOption=None, services=List(), started=false) 2018-11-04 19:56:35,574 INFO [dispatcher-event-loop-1] spark.MapOutputTrackerMasterEndpoint (Logging.scala:logInfo(54)) - MapOutputTrackerMasterEndpoint stopped! 2018-11-04 19:56:35,653 INFO [Driver] memory.MemoryStore (Logging.scala:logInfo(54)) - MemoryStore cleared 2018-11-04 19:56:35,656 INFO [Driver] storage.BlockManager (Logging.scala:logInfo(54)) - BlockManager stopped 2018-11-04 19:56:35,668 INFO [Driver] storage.BlockManagerMaster (Logging.scala:logInfo(54)) - BlockManagerMaster stopped 2018-11-04 19:56:35,678 INFO [dispatcher-event-loop-1] scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint (Logging.scala:logInfo(54)) - OutputCommitCoordinator stopped! 2018-11-04 19:56:35,688 INFO [Driver] spark.SparkContext (Logging.scala:logInfo(54)) - Successfully stopped SparkContext 2018-11-04 19:56:35,689 INFO [Driver] spark.SparkContext (Logging.scala:logInfo(54)) - SparkContext already stopped. 2018-11-04 19:56:35,695 ERROR [Driver] yarn.ApplicationMaster (Logging.scala:logError(91)) - User class threw exception: org.apache.spark.SparkException: Spark test failed: org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: file:/tmp/radoop/training-vm/tmp_1541357744748_x0migqc org.apache.spark.SparkException: Spark test failed: org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: file:/tmp/radoop/training-vm/tmp_1541357744748_x0migqc at eu.radoop.spark.SparkTestCountJobRunner.main(SparkTestCountJobRunner.java:54) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:627) 2018-11-04 19:56:35,708 INFO [Driver] yarn.ApplicationMaster (Logging.scala:logInfo(54)) - Final app status: FAILED, exitCode: 15, (reason: User class threw exception: org.apache.spark.SparkException: Spark test failed: org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: file:/tmp/radoop/training-vm/tmp_1541357744748_x0migqc) 2018-11-04 19:56:35,742 INFO [Thread-2] yarn.ApplicationMaster (Logging.scala:logInfo(54)) - Unregistering ApplicationMaster with FAILED (diag message: User class threw exception: org.apache.spark.SparkException: Spark test failed: org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: file:/tmp/radoop/training-vm/tmp_1541357744748_x0migqc) 2018-11-04 19:56:35,761 INFO [Thread-2] impl.AMRMClientImpl (AMRMClientImpl.java:unregisterApplicationMaster(382)) - Waiting for application to be successfully unregistered. 2018-11-04 19:56:35,871 INFO [Thread-2] yarn.ApplicationMaster (Logging.scala:logInfo(54)) - Deleting staging directory hdfs://192.168.99.100:9020/home/training-vm/.sparkStaging/application_1541356129398_0005 2018-11-04 19:56:35,872 ERROR [Thread-2] util.Utils (Logging.scala:logError(91)) - Uncaught exception in thread Thread-2 java.lang.IllegalArgumentException: Wrong FS: hdfs://192.168.99.100:9020/home/training-vm/.sparkStaging/application_1541356129398_0005, expected: file:/// at org.apache.hadoop.fs.FileSystem.checkPath(FileSystem.java:647) at org.apache.hadoop.fs.RawLocalFileSystem.pathToFile(RawLocalFileSystem.java:82) at org.apache.hadoop.fs.RawLocalFileSystem.deprecatedGetFileStatus(RawLocalFileSystem.java:604) at org.apache.hadoop.fs.RawLocalFileSystem.getFileLinkStatusInternal(RawLocalFileSystem.java:822) at org.apache.hadoop.fs.RawLocalFileSystem.getFileStatus(RawLocalFileSystem.java:599) at org.apache.hadoop.fs.ChecksumFileSystem.delete(ChecksumFileSystem.java:538) at org.apache.spark.deploy.yarn.ApplicationMaster.org$apache$spark$deploy$yarn$ApplicationMaster$$cleanupStagingDir(ApplicationMaster.scala:515) at org.apache.spark.deploy.yarn.ApplicationMaster$$anonfun$run$1.apply$mcV$sp(ApplicationMaster.scala:225) at org.apache.spark.util.SparkShutdownHook.run(ShutdownHookManager.scala:215) at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(ShutdownHookManager.scala:187) at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:187) at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(ShutdownHookManager.scala:187) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1857) at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(ShutdownHookManager.scala:187) at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:187) at org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(ShutdownHookManager.scala:187) at scala.util.Try$.apply(Try.scala:192) at org.apache.spark.util.SparkShutdownHookManager.runAll(ShutdownHookManager.scala:187) at org.apache.spark.util.SparkShutdownHookManager$$anon$2.run(ShutdownHookManager.scala:177) at org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54) 2018-11-04 19:56:35,877 INFO [Thread-2] util.ShutdownHookManager (Logging.scala:logInfo(54)) - Shutdown hook called 2018-11-04 19:56:35,879 INFO [Thread-2] util.ShutdownHookManager (Logging.scala:logInfo(54)) - Deleting directory /tmp/hadoop-training-vm/nm-local-dir/usercache/training-vm/appcache/application_1541356129398_0005/spark-83539307-fc4e-4ad0-a16c-96d34d141c96