<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  <parent>
    <artifactId>spark-parent_2.13</artifactId>
    <groupId>org.apache.spark</groupId>
    <version>3.4.2</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
  <artifactId>spark-hadoop-cloud_2.13</artifactId>
  <name>Spark Project Hadoop Cloud Integration</name>
  <description>Contains Hadoop JARs and transitive dependencies needed to interact with cloud infrastructures.</description>
  <build>
    <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
    <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
  </build>
  <profiles>
    <profile>
      <id>hadoop-2</id>
    </profile>
    <profile>
      <id>hadoop-3</id>
      <build>
        <plugins>
          <plugin>
            <groupId>org.scalatest</groupId>
            <artifactId>scalatest-maven-plugin</artifactId>
            <version>2.2.0</version>
            <executions>
              <execution>
                <id>test</id>
                <phase>test</phase>
                <goals>
                  <goal>test</goal>
                </goals>
                <configuration>
                  <tagsToExclude>org.apache.spark.internal.io.cloud.IntegrationTestSuite</tagsToExclude>
                  <reportsDirectory>/opt/spark-rm/output/spark/hadoop-cloud/target/surefire-reports</reportsDirectory>
                  <junitxml>.</junitxml>
                  <filereports>SparkTestSuite.txt</filereports>
                  <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=128m -XX:+IgnoreUnrecognizedVMOptions
      --add-opens=java.base/java.lang=ALL-UNNAMED
      --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
      --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
      --add-opens=java.base/java.io=ALL-UNNAMED
      --add-opens=java.base/java.net=ALL-UNNAMED
      --add-opens=java.base/java.nio=ALL-UNNAMED
      --add-opens=java.base/java.util=ALL-UNNAMED
      --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
      --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
      --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
      --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
      --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
      --add-opens=java.base/sun.security.action=ALL-UNNAMED
      --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
      -Djdk.reflect.useDirectMethodHandle=false -Dio.netty.tryReflectionSetAccessible=true</argLine>
                  <stderr />
                  <environmentVariables>
                    <SPARK_DIST_CLASSPATH>${test_classpath}</SPARK_DIST_CLASSPATH>
                    <SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES>
                    <SPARK_SCALA_VERSION>2.13</SPARK_SCALA_VERSION>
                    <SPARK_TESTING>1</SPARK_TESTING>
                    <JAVA_HOME>/usr</JAVA_HOME>
                  </environmentVariables>
                  <systemProperties>
                    <log4j.configurationFile>file:src/test/resources/log4j2.properties</log4j.configurationFile>
                    <derby.system.durability>test</derby.system.durability>
                    <java.awt.headless>true</java.awt.headless>
                    <java.io.tmpdir>/opt/spark-rm/output/spark/hadoop-cloud/target/tmp</java.io.tmpdir>
                    <spark.test.home>${session.executionRootDirectory}</spark.test.home>
                    <spark.testing>1</spark.testing>
                    <spark.ui.enabled>false</spark.ui.enabled>
                    <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
                    <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
                    <spark.test.webdriver.chrome.driver />
                    <spark.test.docker.keepContainer>false</spark.test.docker.keepContainer>
                    <spark.test.docker.removePulledImage>true</spark.test.docker.removePulledImage>
                    <test.src.tables>__not_used__</test.src.tables>
                    <spark.hadoop.hadoop.security.key.provider.path>test:///</spark.hadoop.hadoop.security.key.provider.path>
                  </systemProperties>
                  <tagsToInclude />
                </configuration>
              </execution>
            </executions>
            <configuration>
              <reportsDirectory>/opt/spark-rm/output/spark/hadoop-cloud/target/surefire-reports</reportsDirectory>
              <junitxml>.</junitxml>
              <filereports>SparkTestSuite.txt</filereports>
              <argLine>-ea -Xmx4g -Xss4m -XX:MaxMetaspaceSize=2g -XX:ReservedCodeCacheSize=128m -XX:+IgnoreUnrecognizedVMOptions
      --add-opens=java.base/java.lang=ALL-UNNAMED
      --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
      --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
      --add-opens=java.base/java.io=ALL-UNNAMED
      --add-opens=java.base/java.net=ALL-UNNAMED
      --add-opens=java.base/java.nio=ALL-UNNAMED
      --add-opens=java.base/java.util=ALL-UNNAMED
      --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
      --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
      --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
      --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
      --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
      --add-opens=java.base/sun.security.action=ALL-UNNAMED
      --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
      -Djdk.reflect.useDirectMethodHandle=false -Dio.netty.tryReflectionSetAccessible=true</argLine>
              <stderr />
              <environmentVariables>
                <SPARK_DIST_CLASSPATH>${test_classpath}</SPARK_DIST_CLASSPATH>
                <SPARK_PREPEND_CLASSES>1</SPARK_PREPEND_CLASSES>
                <SPARK_SCALA_VERSION>2.13</SPARK_SCALA_VERSION>
                <SPARK_TESTING>1</SPARK_TESTING>
                <JAVA_HOME>/usr</JAVA_HOME>
              </environmentVariables>
              <systemProperties>
                <log4j.configurationFile>file:src/test/resources/log4j2.properties</log4j.configurationFile>
                <derby.system.durability>test</derby.system.durability>
                <java.awt.headless>true</java.awt.headless>
                <java.io.tmpdir>/opt/spark-rm/output/spark/hadoop-cloud/target/tmp</java.io.tmpdir>
                <spark.test.home>${session.executionRootDirectory}</spark.test.home>
                <spark.testing>1</spark.testing>
                <spark.ui.enabled>false</spark.ui.enabled>
                <spark.ui.showConsoleProgress>false</spark.ui.showConsoleProgress>
                <spark.unsafe.exceptionOnMemoryLeak>true</spark.unsafe.exceptionOnMemoryLeak>
                <spark.test.webdriver.chrome.driver />
                <spark.test.docker.keepContainer>false</spark.test.docker.keepContainer>
                <spark.test.docker.removePulledImage>true</spark.test.docker.removePulledImage>
                <test.src.tables>__not_used__</test.src.tables>
                <spark.hadoop.hadoop.security.key.provider.path>test:///</spark.hadoop.hadoop.security.key.provider.path>
              </systemProperties>
              <tagsToExclude>,org.apache.spark.tags.ChromeUITest</tagsToExclude>
              <tagsToInclude />
            </configuration>
          </plugin>
          <plugin>
            <groupId>org.codehaus.mojo</groupId>
            <artifactId>build-helper-maven-plugin</artifactId>
            <version>3.2.0</version>
            <executions>
              <execution>
                <id>module-timestamp-property</id>
                <phase>validate</phase>
                <goals>
                  <goal>timestamp-property</goal>
                </goals>
                <configuration>
                  <name>module.build.timestamp</name>
                  <pattern>yyyy-MM-dd HH:mm:ss z</pattern>
                  <timeSource>current</timeSource>
                  <timeZone>America/Los_Angeles</timeZone>
                </configuration>
              </execution>
              <execution>
                <id>local-timestamp-property</id>
                <phase>validate</phase>
                <goals>
                  <goal>timestamp-property</goal>
                </goals>
                <configuration>
                  <name>local.build.timestamp</name>
                  <pattern>yyyy-MM-dd HH:mm:ss z</pattern>
                  <timeSource>build</timeSource>
                  <timeZone>America/Los_Angeles</timeZone>
                </configuration>
              </execution>
              <execution>
                <id>add-extra-sources</id>
                <phase>generate-sources</phase>
                <goals>
                  <goal>add-source</goal>
                </goals>
                <configuration>
                  <sources>
                    <source>src/hadoop-3/main/java</source>
                    <source>src/hadoop-3/main/scala</source>
                  </sources>
                </configuration>
              </execution>
              <execution>
                <id>add-extra-test-sources</id>
                <phase>generate-test-sources</phase>
                <goals>
                  <goal>add-test-source</goal>
                </goals>
                <configuration>
                  <sources>
                    <source>src/hadoop-3/test/java</source>
                    <source>src/hadoop-3/test/scala</source>
                  </sources>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
      <dependencies>
        <dependency>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-cloud-storage</artifactId>
          <version>3.3.4</version>
          <scope>compile</scope>
          <exclusions>
            <exclusion>
              <artifactId>hadoop-common</artifactId>
              <groupId>org.apache.hadoop</groupId>
            </exclusion>
            <exclusion>
              <artifactId>jackson-core</artifactId>
              <groupId>com.fasterxml.jackson.core</groupId>
            </exclusion>
            <exclusion>
              <artifactId>guava</artifactId>
              <groupId>com.google.guava</groupId>
            </exclusion>
            <exclusion>
              <artifactId>org.jacoco.agent</artifactId>
              <groupId>org.jacoco</groupId>
            </exclusion>
          </exclusions>
        </dependency>
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-util</artifactId>
          <version>9.4.50.v20221201</version>
          <scope>compile</scope>
        </dependency>
        <dependency>
          <groupId>org.eclipse.jetty</groupId>
          <artifactId>jetty-util-ajax</artifactId>
          <version>9.4.50.v20221201</version>
          <scope>compile</scope>
        </dependency>
      </dependencies>
      <properties>
        <os.detected.name>linux</os.detected.name>
        <extra.scala.testsource.dir>src/hadoop-3/test/scala</extra.scala.testsource.dir>
        <extra.java.testsource.dir>src/hadoop-3/test/java</extra.java.testsource.dir>
        <extra.java.source.dir>src/hadoop-3/main/java</extra.java.source.dir>
        <os.detected.arch>x86_64</os.detected.arch>
        <os.detected.release.version>20.04</os.detected.release.version>
        <os.detected.classifier>linux-x86_64</os.detected.classifier>
        <os.detected.release.like.ubuntu>true</os.detected.release.like.ubuntu>
        <os.detected.release.like.debian>true</os.detected.release.like.debian>
        <extra.scala.source.dir>src/hadoop-3/main/scala</extra.scala.source.dir>
        <os.detected.release>ubuntu</os.detected.release>
      </properties>
    </profile>
    <profile>
      <id>integration-test</id>
      <build>
        <plugins>
          <plugin>
            <groupId>org.scalatest</groupId>
            <artifactId>scalatest-maven-plugin</artifactId>
            <executions>
              <execution>
                <id>test</id>
                <phase>test</phase>
                <goals>
                  <goal>test</goal>
                </goals>
                <configuration>
                  <tagsToExclude>None</tagsToExclude>
                  <tagsToInclude>org.apache.spark.internal.io.cloud.IntegrationTestSuite</tagsToInclude>
                </configuration>
              </execution>
            </executions>
          </plugin>
        </plugins>
      </build>
    </profile>
  </profiles>
  <dependencies>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_2.13</artifactId>
      <version>3.4.2</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-sql_2.13</artifactId>
      <version>3.4.2</version>
      <type>test-jar</type>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.spark</groupId>
      <artifactId>spark-core_2.13</artifactId>
      <version>3.4.2</version>
      <type>test-jar</type>
      <scope>test</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client-api</artifactId>
      <version>3.3.4</version>
      <scope>provided</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-client-runtime</artifactId>
      <version>3.3.4</version>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-aws</artifactId>
      <version>3.3.4</version>
      <scope>compile</scope>
      <exclusions>
        <exclusion>
          <artifactId>hadoop-common</artifactId>
          <groupId>org.apache.hadoop</groupId>
        </exclusion>
        <exclusion>
          <artifactId>commons-logging</artifactId>
          <groupId>commons-logging</groupId>
        </exclusion>
        <exclusion>
          <artifactId>jackson-core</artifactId>
          <groupId>com.fasterxml.jackson.core</groupId>
        </exclusion>
        <exclusion>
          <artifactId>jackson-databind</artifactId>
          <groupId>com.fasterxml.jackson.core</groupId>
        </exclusion>
        <exclusion>
          <artifactId>jackson-annotations</artifactId>
          <groupId>com.fasterxml.jackson.core</groupId>
        </exclusion>
        <exclusion>
          <artifactId>aws-java-sdk</artifactId>
          <groupId>com.amazonaws</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-openstack</artifactId>
      <version>3.3.4</version>
      <scope>compile</scope>
      <exclusions>
        <exclusion>
          <artifactId>hadoop-common</artifactId>
          <groupId>org.apache.hadoop</groupId>
        </exclusion>
        <exclusion>
          <artifactId>commons-logging</artifactId>
          <groupId>commons-logging</groupId>
        </exclusion>
        <exclusion>
          <artifactId>junit</artifactId>
          <groupId>junit</groupId>
        </exclusion>
        <exclusion>
          <artifactId>mockito-all</artifactId>
          <groupId>org.mockito</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>com.google.cloud.bigdataoss</groupId>
      <artifactId>gcs-connector</artifactId>
      <version>hadoop3-2.2.11</version>
      <classifier>shaded</classifier>
      <scope>compile</scope>
      <exclusions>
        <exclusion>
          <artifactId>*</artifactId>
          <groupId>*</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>joda-time</groupId>
      <artifactId>joda-time</artifactId>
      <version>2.12.2</version>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-databind</artifactId>
      <version>2.14.2</version>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.core</groupId>
      <artifactId>jackson-annotations</artifactId>
      <version>2.14.2</version>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>com.fasterxml.jackson.dataformat</groupId>
      <artifactId>jackson-dataformat-cbor</artifactId>
      <version>2.14.2</version>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.httpcomponents</groupId>
      <artifactId>httpclient</artifactId>
      <version>4.5.14</version>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.httpcomponents</groupId>
      <artifactId>httpcore</artifactId>
      <version>4.4.16</version>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-azure</artifactId>
      <version>3.3.4</version>
      <scope>compile</scope>
      <exclusions>
        <exclusion>
          <artifactId>hadoop-common</artifactId>
          <groupId>org.apache.hadoop</groupId>
        </exclusion>
        <exclusion>
          <artifactId>jackson-core</artifactId>
          <groupId>com.fasterxml.jackson.core</groupId>
        </exclusion>
        <exclusion>
          <artifactId>guava</artifactId>
          <groupId>com.google.guava</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.apache.hadoop</groupId>
      <artifactId>hadoop-cloud-storage</artifactId>
      <version>3.3.4</version>
      <scope>compile</scope>
      <exclusions>
        <exclusion>
          <artifactId>hadoop-common</artifactId>
          <groupId>org.apache.hadoop</groupId>
        </exclusion>
        <exclusion>
          <artifactId>jackson-core</artifactId>
          <groupId>com.fasterxml.jackson.core</groupId>
        </exclusion>
        <exclusion>
          <artifactId>guava</artifactId>
          <groupId>com.google.guava</groupId>
        </exclusion>
        <exclusion>
          <artifactId>org.jacoco.agent</artifactId>
          <groupId>org.jacoco</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.eclipse.jetty</groupId>
      <artifactId>jetty-util</artifactId>
      <version>9.4.50.v20221201</version>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>org.eclipse.jetty</groupId>
      <artifactId>jetty-util-ajax</artifactId>
      <version>9.4.50.v20221201</version>
      <scope>compile</scope>
    </dependency>
    <dependency>
      <groupId>org.scalatest</groupId>
      <artifactId>scalatest_2.13</artifactId>
      <version>3.2.15</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>scalatest-core_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-featurespec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-flatspec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-freespec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-funsuite_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-funspec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-propspec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-refspec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-wordspec_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-diagrams_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-matchers-core_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-shouldmatchers_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-mustmatchers_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.scalatestplus</groupId>
      <artifactId>scalacheck-1-17_2.13</artifactId>
      <version>3.2.15.0</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>scalacheck_2.13</artifactId>
          <groupId>org.scalacheck</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-core_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.scalatestplus</groupId>
      <artifactId>mockito-4-6_2.13</artifactId>
      <version>3.2.15.0</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>mockito-core</artifactId>
          <groupId>org.mockito</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-core_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>org.scalatestplus</groupId>
      <artifactId>selenium-4-7_2.13</artifactId>
      <version>3.2.15.0</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>selenium-java</artifactId>
          <groupId>org.seleniumhq.selenium</groupId>
        </exclusion>
        <exclusion>
          <artifactId>htmlunit-driver</artifactId>
          <groupId>org.seleniumhq.selenium</groupId>
        </exclusion>
        <exclusion>
          <artifactId>scalatest-core_2.13</artifactId>
          <groupId>org.scalatest</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>junit</groupId>
      <artifactId>junit</artifactId>
      <version>4.13.2</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>hamcrest-core</artifactId>
          <groupId>org.hamcrest</groupId>
        </exclusion>
      </exclusions>
    </dependency>
    <dependency>
      <groupId>com.github.sbt</groupId>
      <artifactId>junit-interface</artifactId>
      <version>0.13.3</version>
      <scope>test</scope>
      <exclusions>
        <exclusion>
          <artifactId>test-interface</artifactId>
          <groupId>org.scala-sbt</groupId>
        </exclusion>
      </exclusions>
    </dependency>
  </dependencies>
  <properties>
    <sbt.project.name>hadoop-cloud</sbt.project.name>
  </properties>
</project>
