From 1e4ee297f2437fd6c6f6782b864c6f9b3eed2b7e Mon Sep 17 00:00:00 2001 From: Dustin Jenkins Date: Mon, 20 Oct 2025 15:19:18 -0700 Subject: [PATCH 1/3] feat: added download manager and updated existing applications --- .github/workflows/gradle.yml | 19 +- cadc-app-kit/build.gradle | 7 +- cadc-download-manager-server/build.gradle | 8 +- cadc-download-manager/build.gradle | 6 +- downloadManager/Dockerfile | 6 + downloadManager/README | 14 + downloadManager/VERSION | 9 + downloadManager/build.gradle | 72 ++ .../integration/DownloadManagerIntTest.java | 767 ++++++++++++++++++ .../java/ca/nrc/cadc/dlm/server/SkinUtil.java | 92 +++ .../downloadManagerBundle_en.properties | 88 ++ .../downloadManagerBundle_fr.properties | 97 +++ .../src/main/resources/cadcDownload | 216 +++++ downloadManager/src/main/webapp/WEB-INF/c.tld | 563 +++++++++++++ .../src/main/webapp/WEB-INF/fmt.tld | 712 ++++++++++++++++ .../src/main/webapp/WEB-INF/web.xml | 109 +++ .../test/src/resources/TestPage.html | 184 +++++ gradle/wrapper/gradle-wrapper.properties | 2 +- opencadc.gradle | 74 +- settings.gradle | 2 +- 20 files changed, 3019 insertions(+), 28 deletions(-) create mode 100644 downloadManager/Dockerfile create mode 100644 downloadManager/README create mode 100644 downloadManager/VERSION create mode 100644 downloadManager/build.gradle create mode 100644 downloadManager/src/intTest/java/ca/nrc/cadc/downloadManager/integration/DownloadManagerIntTest.java create mode 100644 downloadManager/src/main/java/ca/nrc/cadc/dlm/server/SkinUtil.java create mode 100644 downloadManager/src/main/resources/ca/nrc/cadc/downloadManager/downloadManagerBundle_en.properties create mode 100644 downloadManager/src/main/resources/ca/nrc/cadc/downloadManager/downloadManagerBundle_fr.properties create mode 100755 downloadManager/src/main/resources/cadcDownload create mode 100644 downloadManager/src/main/webapp/WEB-INF/c.tld create mode 100644 downloadManager/src/main/webapp/WEB-INF/fmt.tld create mode 100644 downloadManager/src/main/webapp/WEB-INF/web.xml create mode 100644 downloadManager/test/src/resources/TestPage.html diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 5818c61..38a338e 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -10,21 +10,18 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Set up JDK 1.8 - uses: actions/setup-java@v3 + - uses: actions/checkout@v5 + - name: Set up JDK 21 + uses: actions/setup-java@v5 with: - java-version: 8 + java-version: 21 distribution: temurin - - name: build and test cadc-app-kit - run: cd cadc-app-kit && ../gradlew --info clean build javadoc checkstyleMain install - - name: build and test cadc-download-manager run: cd cadc-download-manager && ../gradlew --info clean build javadoc checkstyleMain install - - name: build and test cadc-upload-manager - run: cd cadc-upload-manager && ../gradlew --info clean build javadoc checkstyleMain install - - name: build and test cadc-download-manager-server - run: cd cadc-download-manager-server && ../gradlew --info clean build javadoc checkstyleMain install + run: cd cadc-download-manager-server && ../gradlew --info clean build javadoc checkstyleMain install + + - name: build and test downloadManager application + run: cd downloadManager && ../gradlew --info clean build javadoc checkstyleMain diff --git a/cadc-app-kit/build.gradle b/cadc-app-kit/build.gradle index 2cf15b3..5f8d1af 100644 --- a/cadc-app-kit/build.gradle +++ b/cadc-app-kit/build.gradle @@ -1,6 +1,5 @@ plugins { id 'java' - id 'maven' id 'maven-publish' id 'checkstyle' } @@ -12,7 +11,11 @@ repositories { version = '1.0.1' group = 'org.opencadc' -sourceCompatibility = 1.8 + +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} description = 'OpenCADC Swing App Kit' def git_url = 'https://github.com/opencadc/apps' diff --git a/cadc-download-manager-server/build.gradle b/cadc-download-manager-server/build.gradle index 2e8497a..3e9c7a8 100644 --- a/cadc-download-manager-server/build.gradle +++ b/cadc-download-manager-server/build.gradle @@ -1,6 +1,5 @@ plugins { id 'java' - id 'maven' id 'maven-publish' id 'checkstyle' } @@ -12,13 +11,16 @@ repositories { version = '1.6.0' group = 'org.opencadc' -sourceCompatibility = 1.8 + +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} description = 'OpenCADC DownloadManager server library' def git_url = 'https://github.com/opencadc/apps' - dependencies { implementation 'javax.servlet:javax.servlet-api:[3.1,)' diff --git a/cadc-download-manager/build.gradle b/cadc-download-manager/build.gradle index 6569a3c..1300036 100644 --- a/cadc-download-manager/build.gradle +++ b/cadc-download-manager/build.gradle @@ -1,6 +1,5 @@ plugins { id 'java' - id 'maven' id 'maven-publish' id 'application' id 'checkstyle' @@ -13,7 +12,10 @@ repositories { version = '1.5.2' group = 'org.opencadc' -sourceCompatibility = 1.8 +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} description = 'OpenCADC DownloadManager client' def git_url = 'https://github.com/opencadc/apps' diff --git a/downloadManager/Dockerfile b/downloadManager/Dockerfile new file mode 100644 index 0000000..acdbd0e --- /dev/null +++ b/downloadManager/Dockerfile @@ -0,0 +1,6 @@ +FROM images.opencadc.org/library/cadc-tomcat:1.4 + +# Clear out the webapps directory. +RUN rm -rf webapps/* + +COPY build/libs/*.war /usr/share/tomcat/webapps/ diff --git a/downloadManager/README b/downloadManager/README new file mode 100644 index 0000000..6552be6 --- /dev/null +++ b/downloadManager/README @@ -0,0 +1,14 @@ + +known issues: + +"gradle clean build" fails because something done in the configuration phase gets wiped by the clean + +build always unzips and signs the jnlp jar files + +work around: do the clean and build as separate commands + +Running integration tests requires: +- valid test certificates (in $A/test-certificates) +- valid servops.pem +- valid ca.crt file + diff --git a/downloadManager/VERSION b/downloadManager/VERSION new file mode 100644 index 0000000..9f7ee3b --- /dev/null +++ b/downloadManager/VERSION @@ -0,0 +1,9 @@ +## deployable containers have a semantic and build tag +# version tag: major.minor.patch +# build version tag: timestamp +# tags with and without build number so operators use the versioned +# tag but we always keep a timestamped tag in case a semantic tag gets +# replaced accidentally +VER=1.6.0 +TAGS="${VER} ${VER}-$(date -u +"%Y%m%dT%H%M%S")" +unset VER diff --git a/downloadManager/build.gradle b/downloadManager/build.gradle new file mode 100644 index 0000000..8bd2e66 --- /dev/null +++ b/downloadManager/build.gradle @@ -0,0 +1,72 @@ +plugins { + id 'war' + id 'checkstyle' +} + +repositories { + mavenCentral() + mavenLocal() +} + +java { + sourceCompatibility = JavaVersion.VERSION_11 + targetCompatibility = JavaVersion.VERSION_11 +} + +dependencies { + providedCompile 'javax.servlet:javax.servlet-api:3.1.+' + + implementation 'org.apache.commons:commons-configuration2:[2.7,3.0)' + implementation 'org.apache.commons:commons-lang3:[3.18,4.0)' + implementation 'org.opencadc:cadc-access-control-identity:[1.1.0,2.0)' + implementation 'org.opencadc:cadc-dali:[1.2.15,2.0)' + implementation 'org.opencadc:cadc-download-manager-server:[1.6.0,)' + implementation 'org.opencadc:cadc-log:[1.1.1,)' + implementation 'org.opencadc:cadc-util:[1.6.0,2.0)' + + runtimeOnly 'org.opencadc:cadc-access-control-identity:[1.1.0,)' + runtimeOnly 'org.opencadc:cadc-download-manager:1.5.2' + runtimeOnly 'jstl:jstl:1.2' + + testImplementation 'org.opencadc:cadc-download-manager:1.5.2' + testImplementation 'junit:junit:4.+' +} + +ext { + intTest_default_web_app_url = 'https://rc-www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca' + intTest_default_web_app_endpoint = '/download' +} + +war { + from(file('build/tmp')) { + include 'codebase/*.jar' + } +} + +tasks.register('duplicateENWar', Copy) { + from 'build/libs/downloadManager.war' + into 'build/libs' + rename('downloadManager.war', 'en#download.war') +} + +tasks.register('duplicateFRWar', Copy) { + from 'build/libs/downloadManager.war' + into 'build/libs' + rename('downloadManager.war', 'fr#telecharger.war') +} + +configurations { + intTestImplementation.extendsFrom testImplementation + intTestRuntime.extendsFrom testRuntime +} + +tasks.register('compileDeps', DependencyReportTask) { + configurations = [project.configurations.implementation] as Set +} + +duplicateFRWar.dependsOn(duplicateENWar) + +build.finalizedBy(duplicateENWar, duplicateFRWar) +assemble.finalizedBy(duplicateENWar, duplicateFRWar) + +apply from: '../opencadc.gradle' diff --git a/downloadManager/src/intTest/java/ca/nrc/cadc/downloadManager/integration/DownloadManagerIntTest.java b/downloadManager/src/intTest/java/ca/nrc/cadc/downloadManager/integration/DownloadManagerIntTest.java new file mode 100644 index 0000000..176b4ae --- /dev/null +++ b/downloadManager/src/intTest/java/ca/nrc/cadc/downloadManager/integration/DownloadManagerIntTest.java @@ -0,0 +1,767 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2011. (c) 2011. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + * $Revision: 5 $ + * + ************************************************************************ + */ + +package ca.nrc.cadc.downloadManager.integration; + + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.CookiePrincipal; +import ca.nrc.cadc.auth.PrincipalExtractor; +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.auth.X509CertificateChain; +import ca.nrc.cadc.dlm.DownloadTuple; +import ca.nrc.cadc.dlm.server.DispatcherServlet; +import ca.nrc.cadc.dlm.server.ServerUtil; +import ca.nrc.cadc.net.ContentType; +import ca.nrc.cadc.net.FileContent; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.net.NetUtil; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.reg.client.LocalAuthority; +import ca.nrc.cadc.reg.client.RegistryClient; +import ca.nrc.cadc.util.ArgumentMap; +import ca.nrc.cadc.util.Log4jInit; +import ca.nrc.cadc.xml.JsonInputter; +import ca.nrc.cadc.xml.XmlUtil; +import java.nio.charset.Charset; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.jdom2.Document; +import org.jdom2.Element; +import org.junit.Assert; +import org.junit.Test; + +import javax.security.auth.Subject; +import java.io.ByteArrayOutputStream; +import java.io.LineNumberReader; +import java.io.StringReader; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.security.Principal; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * @author pdowler + */ +public class DownloadManagerIntTest { + private static final Logger log = Logger.getLogger(DownloadManagerIntTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.downloadManager", Level.DEBUG); + } + + private final static URI DOWNLOAD_URI = URI.create("ivo://cadc.nrc.ca/download"); + + private final static String PUBLIC_CAOM_URI = "ivo://cadc.nrc.ca/BLAST?BLASTabell31122006-12-21/REDUCED_250_2009-03-06"; + + private final static String SI_JCMT_URI = "cadc:JCMT/a20210525_00032_01_0001.sdf"; + + // proprietary until 2031-01-01 aka permanently + // - if this test is failing, we may need to find another proprietary plane + private final static String PROP_CAOM_URI = "ivo://cadc.nrc.ca/JCMT?scuba2_00024_20121023T034337/raw-850um"; + + private final static String PUBLIC_SUBARU_URI = "ivo://cadc.nrc.ca/maq/SUBARU?SUPE01334570/SUPA0133457X"; + + // For testing JSON payload to web service + private static String URI_STR = "test://mysite.ca/path/1"; + private static String SHAPE_STR = "polygon 0 0 0 0"; + private static String LABEL_STR = "label"; + + // Using Badgerfish json so that JsonInputter can read it correctly + private static String TUPLE_JSON_URI = "{\"tupleID\":\"{\"$\":\"" + URI_STR + "\"}}"; + private static String TUPLE_JSON_SHAPE = "{\"tupleID\":\"{\"$\":\"" + URI_STR + "\"},\"shape\":\"{\"$\":\"" + SHAPE_STR + "\"}}"; + + private static String TUPLE_JSON = "{\"tuple\":{\"tupleID\":{\"$\":\"" + URI_STR + "\"}," + + "\"shape\":{\"$\":\"" + SHAPE_STR + "\"}," + + "\"label\":{\"$\":\"" + LABEL_STR + "\"}}}"; + + Subject anonSubject; + Subject authSubject; + RegistryClient reg; + + public DownloadManagerIntTest() { + this.reg = new RegistryClient(); + anonSubject = AuthenticationUtil.getSubject(new PrincipalExtractor() { + + public Set getPrincipals() { + return new TreeSet<>(); + } + + public X509CertificateChain getCertificateChain() { + return null; + } + }); + + authSubject = AuthenticationUtil.getSubject(new PrincipalExtractor() { + Principal cookie = getSSOCookie(); + + public Set getPrincipals() { + Set ret = new HashSet(); + ret.add(cookie); + return ret; + } + + public X509CertificateChain getCertificateChain() { + return null; + } + + // obsolete + private CookiePrincipal getSSOCookie() { + Map params = new TreeMap<>(); + params.put("username", "cadcregtest1"); + params.put("password", "qS1U42Y"); + + LocalAuthority localAuthority = new LocalAuthority(); + URI serviceURI = localAuthority.getServiceURI(Standards.UMS_LOGIN_01.toString()); + log.debug("login uri: " + serviceURI.toString()); + URL url = reg.getServiceURL(serviceURI, Standards.UMS_LOGIN_01, AuthMethod.ANON); + log.debug("login url: " + url.toExternalForm()); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost login = new HttpPost(url, params, bos); + login.run(); + if (login.getThrowable() != null) { + throw new RuntimeException("login failed: " + login.getResponseCode(), login.getThrowable()); + } + String token = bos.toString(); + return new CookiePrincipal("CADC_SSO", token); + } + }); + } + + public String getDomainName(String hostname) + { + String s = hostname; + s = s.trim(); + int i = s.indexOf('.'); + if (i > 0) + return s.substring(i+1); + else + return s; + } + + @Test + public void testAnonHTML() { + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + Map params = new TreeMap(); + params.put("uri", PUBLIC_CAOM_URI); + params.put("method", DispatcherServlet.HTMLLIST); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + Subject.doAs(anonSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + Assert.assertNotNull(post.getResponseContentType()); + ContentType ct = new ContentType(post.getResponseContentType()); + Assert.assertEquals("text/html", ct.getBaseType()); + + String html = bos.toString(); + log.debug("testAnonHTML:\n" + html); + + // TODO: parse and verify URLs are present? other format tests check to different levels + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAnonStorageInventory() { + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + Map params = new TreeMap(); + params.put("uri", SI_JCMT_URI); + params.put("method", DispatcherServlet.SHELL_SCRIPT); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + Subject.doAs(anonSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + Assert.assertNotNull(post.getResponseContentType()); + ContentType ct = new ContentType(post.getResponseContentType()); + Assert.assertEquals("text/x-shellscript", ct.getBaseType()); + + String shellScript = bos.toString(); + log.debug("testAnonScript:\n" + shellScript); + + // TODO: parse and verify URLs are present? other format tests check to different levels + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAuthHTML() { + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + Map params = new TreeMap(); + params.put("uri", PROP_CAOM_URI); + params.put("method", DispatcherServlet.HTMLLIST); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + Subject.doAs(authSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + Assert.assertNotNull(post.getResponseContentType()); + ContentType ct = new ContentType(post.getResponseContentType()); + Assert.assertEquals("text/html", ct.getBaseType()); + + String html = bos.toString(); + log.debug("testAuthHTML:\n" + html); + + // TODO: parse and verify URLs are present? other format tests check to different levels + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAnonTextPlain() { + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + Map params = new TreeMap<>(); + params.put("uri", PUBLIC_CAOM_URI); + params.put("method", DispatcherServlet.URLS); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + Subject.doAs(anonSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + Assert.assertNotNull(post.getResponseContentType()); + ContentType ct = new ContentType(post.getResponseContentType()); + Assert.assertEquals("text/plain", ct.getBaseType()); + + String doc = bos.toString(); + log.debug("testAnonTextPlain:\n" + doc); + + LineNumberReader r = new LineNumberReader(new StringReader(doc)); + String line = r.readLine(); + while (line != null) { + line = line.trim(); + String[] tokens = line.split("[\\s]"); + Assert.assertEquals("tokens on line [" + line + "]", 1, tokens.length); + Assert.assertNotSame("ERROR", tokens[0]); + try { + URL dl = new URL(tokens[0]); + } catch (MalformedURLException bad) { + Assert.fail("invalid URL: " + tokens[0] + " " + bad); + } + + line = r.readLine(); + } + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAuthTextPlain() { + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + Map params = new TreeMap(); + params.put("uri", PROP_CAOM_URI); + params.put("method", DispatcherServlet.URLS); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + Subject.doAs(authSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + Assert.assertNotNull(post.getResponseContentType()); + ContentType ct = new ContentType(post.getResponseContentType()); + Assert.assertEquals("text/plain", ct.getBaseType()); + + String doc = bos.toString(); + log.debug("testAuthTextPlain:\n" + doc); + + LineNumberReader r = new LineNumberReader(new StringReader(doc)); + String line = r.readLine(); + while (line != null) { + line = line.trim(); + // authSubject can't see proprietary metadata so: ERROR until we + // have content to support auth test + assertNotFoundError(line, PROP_CAOM_URI); + //assertValidURL(line, PROP_CAOM_URI); + line = r.readLine(); + } + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + private void assertValidURL(String line, String uri) { + String[] tokens = line.split("[\\s]"); + try { + URL u = new URL(tokens[0]); + log.debug("valid: " + u); + } catch (MalformedURLException bad) { + Assert.fail("invalid URL: " + tokens[0] + " " + bad); + } + } + + private void assertUnauthorized(final String line, final String uri) { + String[] tokens = line.split("[\\t]"); + Assert.assertEquals("tokens on line", 3, tokens.length); + Assert.assertEquals("ERROR", tokens[0]); + Assert.assertEquals(uri, tokens[1]); + // 2-3 is the error message from the datalink service + Assert.assertEquals("failed to resolve URI: authentication failed (401) Unauthorized", tokens[2]); +// Assert.assertEquals(uri, tokens[3]); + } + + private void assertNotFoundError(String line, String uri) { + String[] tokens = line.split("[\\s]"); + Assert.assertEquals("tokens on line", 4, tokens.length); + Assert.assertEquals("ERROR", tokens[0]); + Assert.assertEquals(uri, tokens[1]); + // 2-3 is the error message from the datalink service + Assert.assertEquals("NotFoundFault:", tokens[2]); + Assert.assertEquals(uri, tokens[3]); + } + + @Test + public void testAnonProprietary() throws Exception { + // this test will fail if PROP_CAOM_URI ever becomes public or disappears + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + Map params = new TreeMap(); + params.put("uri", PROP_CAOM_URI); + params.put("method", DispatcherServlet.URLS); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + Subject.doAs(anonSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + Assert.assertNotNull(post.getResponseContentType()); + ContentType ct = new ContentType(post.getResponseContentType()); + Assert.assertEquals("text/plain", ct.getBaseType()); + + String doc = bos.toString(); + log.debug("testAnonProprietary:\n" + doc); + + LineNumberReader r = new LineNumberReader(new StringReader(doc)); + String line = r.readLine(); + while (line != null) { + line = line.trim(); + assertNotFoundError(line, PROP_CAOM_URI); + line = r.readLine(); + } + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + throw unexpected; + } + } + + @Test + public void testAnonJNLP() { + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + Map params = new TreeMap(); + params.put("uri", PUBLIC_CAOM_URI); + params.put("method", DispatcherServlet.WEBSTART); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + Subject.doAs(anonSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + Assert.assertNotNull(post.getResponseContentType()); + ContentType ct = new ContentType(post.getResponseContentType()); + Assert.assertEquals("application/x-java-jnlp-file", ct.getBaseType()); + + // DownloadManager.jsp processing puts some extraneous whitespace at + // the beginning of the document which is technically not allowed by XML, + // so: trim() + String xml = bos.toString().trim(); + log.debug("testAnonJNLP:\n" + xml); + + // TODO: parse and verify URI + Document doc = XmlUtil.buildDocument(xml); + Assert.assertNotNull("document", doc); + Element jnlp = doc.getRootElement(); + Assert.assertNotNull("jnlp", jnlp); + Element app = jnlp.getChild("application-desc"); + Assert.assertNotNull("Application-desc", app); + List args = app.getChildren("argument"); + + // Uncomment to force an empty argument failure + //args.add(new Element("argument")); + Assert.assertNotNull("arguments", args); + Assert.assertFalse("empty args", args.isEmpty()); + + String[] cmdlineArgs = new String[args.size()]; + for (int i = 0; i < args.size(); i++) { + String arg = args.get(i).getTextTrim(); + Assert.assertFalse("empty argument", arg.isEmpty()); + cmdlineArgs[i] = arg; + } + ArgumentMap am = new ArgumentMap(cmdlineArgs); + boolean verbose = am.isSet("verbose"); + boolean cookie = am.isSet("ssocookie"); + boolean domain = am.isSet("ssocookiedomain"); + Assert.assertTrue("verbose", verbose); + Assert.assertFalse("ssocookie", cookie); + Assert.assertFalse("ssocookiedomain", domain); + + // URIs now passed as plain positional args + //Assert.assertEquals(PUBLIC_CAOM_URI, am.getValue("uris")); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + /** + * Only works in Production. + * @throws Exception Anything went wrong. + */ + @Test + public void testAuthJNLP() throws Exception { + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + if (!url.getHost().equals("www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca")) { + log.warn("Skipping testAuthJNLP due to Access Control issue when not running on Production."); + return; + } + + Map params = new TreeMap(); + params.put("uri", PROP_CAOM_URI); + params.put("method", DispatcherServlet.WEBSTART); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + Subject.doAs(authSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + Assert.assertNotNull(post.getResponseContentType()); + ContentType ct = new ContentType(post.getResponseContentType()); + Assert.assertEquals("application/x-java-jnlp-file", ct.getBaseType()); + + // trim() : see above + String xml = bos.toString().trim(); + log.debug("testAnonJNLP:\n" + xml); + + // TODO: parse and verify URI + Document doc = XmlUtil.buildDocument(xml); + Assert.assertNotNull("document", doc); + Element jnlp = doc.getRootElement(); + Assert.assertNotNull("jnlp", jnlp); + Element app = jnlp.getChild("application-desc"); + Assert.assertNotNull("Application-desc", app); + List args = app.getChildren("argument"); + Assert.assertNotNull("arguments", args); + Assert.assertFalse("empty args", args.isEmpty()); + + String[] cmdlineArgs = new String[args.size()]; + for (int i = 0; i < args.size(); i++) { + cmdlineArgs[i] = args.get(i).getTextTrim(); + } + ArgumentMap am = new ArgumentMap(cmdlineArgs); + boolean verbose = am.isSet("verbose"); + boolean cookie = am.isSet("ssocookie"); + boolean domain = am.isSet("ssocookiedomain"); + Assert.assertTrue("verbose", verbose); + Assert.assertTrue("ssocookie", cookie); + Assert.assertTrue("ssocookiedomain", domain); + + // URIs now passed as plain positional args + //Assert.assertEquals(PROP_CAOM_URI, am.getValue("uris")); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + throw unexpected; + } + } + + @Test + public void testResolveSUBARU() throws Exception { + final URL serviceURL = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + final URL url = new URL(serviceURL.getProtocol() + "://" + serviceURL.getHost() + + "/downloadManager/download"); + + Map params = new TreeMap<>(); + params.put("uri", PUBLIC_SUBARU_URI); + params.put("method", DispatcherServlet.URLS); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + post.run(); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + Assert.assertNotNull(post.getResponseContentType()); + ContentType ct = new ContentType(post.getResponseContentType()); + Assert.assertEquals("text/plain", ct.getBaseType()); + + String doc = bos.toString(); + log.debug("testAuthTextPlain:\n" + doc); + + LineNumberReader r = new LineNumberReader(new StringReader(doc)); + String line = r.readLine(); + + if (line == null) { + Assert.fail("Should have content."); + } else { + while (line != null) { + line = line.trim(); + Assert.assertEquals("tokens on line [" + line + "]", 1, line.split("[\\s]").length); + line = r.readLine(); + } + } + } + + + @Test + public void testLoadChooserURIs() { + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + Map params = new TreeMap(); + params.put("uri", PUBLIC_CAOM_URI); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + Subject.doAs(authSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + // This tests that the page loads, that's about it. + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testLoadChooserURIsPlusPOSCutout() { + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + Map params = new TreeMap(); + params.put("uri", PUBLIC_CAOM_URI); + params.put("pos", NetUtil.encode("circle 9.0 8.0 0.5")); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + Subject.doAs(authSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + // This tests that the page loads, that's about it. + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testLoadChooserURIsPlusPixelCutout() { + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + Map params = new TreeMap(); + params.put("uri", PUBLIC_CAOM_URI); + params.put("cutout", NetUtil.encode("[2]")); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost post = new HttpPost(url, params, bos); + post.setFollowRedirects(false); + Subject.doAs(authSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + // This tests that the page loads, that's about it. + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + + @Test + public void testJSONTupleInput() throws Exception { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + // pass in just tuples, this can/should be supported as well as multi-part data by the + // underlying SyncInput class + log.debug("testJSONTupleInput"); + // Build an array of one + String jsonTuples = "{\"tupleList\":{\"$\":[" + TUPLE_JSON + "]}}"; + log.debug("jsonTuples string:" + jsonTuples); + + HttpPost post = new HttpPost(url, new FileContent(jsonTuples, "application/json", Charset.forName("UTF-8")), false); + Subject.doAs(anonSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + } + + + @Test + public void testAnonTextPlainJSON() { + try { + URL url = reg.getAccessURL(RegistryClient.Query.APPLICATIONS, DownloadManagerIntTest.DOWNLOAD_URI); + url = new URL(url.getProtocol() + "://" + url.getHost() + "/downloadManager/download"); + + Map params = new TreeMap<>(); + params.put("method", DispatcherServlet.URLS); + params.put("runid", "testRunID"); + + log.debug("testJSONTupleInput"); + // Build an array of one + String jsonTuples = "{\"tupleList\":{\"$\":[" + TUPLE_JSON + "]}}"; + log.debug("jsonTuples string:" + jsonTuples); + params.put("jsonTuples", new FileContent(jsonTuples, "application/json", Charset.forName("UTF-8"))); + + HttpPost post = new HttpPost(url, params, false); + Subject.doAs(anonSubject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals("response code", 200, post.getResponseCode()); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } +} diff --git a/downloadManager/src/main/java/ca/nrc/cadc/dlm/server/SkinUtil.java b/downloadManager/src/main/java/ca/nrc/cadc/dlm/server/SkinUtil.java new file mode 100644 index 0000000..79ed92a --- /dev/null +++ b/downloadManager/src/main/java/ca/nrc/cadc/dlm/server/SkinUtil.java @@ -0,0 +1,92 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2020. (c) 2020. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + * $Revision: 4 $ + * + ************************************************************************ + */ + +package ca.nrc.cadc.dlm.server; + +/** + * Use variables in this class to set headers and footers for the + * Download Manager JSP pages. + * If skinURL is provided, it must utilize the following structure: + * - skinURL/htmlHead + * - skinURL/bodyHeader + * - skinURL/bodyFooter + * If not empty, headerURL and footerURL take precedence over skinURL. + * TODO: Likely not used. Could be destroyed. + */ +public class SkinUtil { + public static String skinURL = ""; + + /* + * TODO: need to get LastChangedDate and requestHeaderLang filtered in + * TODO: String headerURL = "http://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/" + requestHeaderLang + * TODO: + "/_page_header.html?LAST_MOD=$LastChangedDate$"; + */ + public static String headerURL = "https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/en/_page_header.html?LAST_MOD=$LastChangedDate$"; + public static String footerURL = "https://www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/cadc/includes/_page_footer_js.html"; +} diff --git a/downloadManager/src/main/resources/ca/nrc/cadc/downloadManager/downloadManagerBundle_en.properties b/downloadManager/src/main/resources/ca/nrc/cadc/downloadManager/downloadManagerBundle_en.properties new file mode 100644 index 0000000..68b0f66 --- /dev/null +++ b/downloadManager/src/main/resources/ca/nrc/cadc/downloadManager/downloadManagerBundle_en.properties @@ -0,0 +1,88 @@ +# Title and page header +TITLE = Download Manager +DOWNLOAD_LINK = /en/download/ +CAOM_PACKAGE_ENDPOINT = /en/download/package +PAGE_HEADER = Choose one of the following download methods: +DATE_MODIFIED_LABEL = Date Modified + +# Buttons +URL_LIST_BUTTON = URL list in a file +HTML_LIST_BUTTON = URL list on an HTML page +SHELL_SCRIPT_BUTTON = Shell Script access +TAR_PACKAGE_BUTTON = TAR file +ZIP_PACKAGE_BUTTON = ZIP file + +# Download type descriptions +JAVA_WEB_START_DESCRIPTION = DownloadManager is launched as a desktop application via \ +Java Webstart. The software is automatically cached on your computer, so subsequent \ +startups are faster. + +TAR_PACKAGE_DESCRIPTION = Obtain a TAR (.tar extension) file. + +ZIP_PACKAGE_DESCRIPTION = Obtain a ZIP (.zip extension) file. + +SHELL_SCRIPT_DESCRIPTION = Obtain a runnable Bash script to be run on the command line. Intended for \ +users comfortable with the command line. Authenticated (logged in) users will have a Token embedded \ +in the script to persist authentication. This Token will expire after some time after download. \ +

\ +$ . ./cadc-download-xxxx.sh + +URL_LIST_DESCRIPTION = Download a text file containing a list of URLs. It can then be \ +used with a script or directly with the wget command. Each line contains a URL \ +or, when URL generation fails, an error message. \ +The wget command to download all the URLs contained the above text file: \ +
\ +% wget --content-disposition -i FILE_NAME \ +

\ +Please note that only Public files are available through this option at this time. \ +
\ +See below for more wget options. + +HTML_LIST_DESCRIPTION = View the list of URLs (one per file) in a Web page and \ +select individual files to download. + +# Remember download method +REMEMBER_CHECKBOX_LABEL = Remember my choice of download method +REMEMBER_CHECKBOX_TEXT = (cookies required) +REMEMBER_TEXT = Each download page has a \"Choose one of the other download methods\" \ +button which, if selected, removes the remembered download choice and returns to this \ +multiple choice page. + +# Help +HELP_HEADER = Help + +HELP_WGET_NOT_WORKING_HEADER = is not working +HELP_WGET_NOT_WORKING_TEXT = \ +

\ + The recommended usage above includes the --content-disposition option, \ + which is available in wget versions 1.12 or later. This option improves the \ + likelihood that saved files will have the correct filenames when downloaded. \ +

\ +

\ + Please note that there are many versions of wget with a variety of options \ + and syntax. Please consult your local help pages. wget --help \ + should print the options for your version of wget \ +

\ +

\ + The wget command should be available on most systems. If not, wget can \ + be downloaded from gnu.org. \ + Alternately, you can try one of the several other download utilities such as: curl, \ + HTTrack, leech (Mozilla Add-on), pavuk, lftp, etc. \ +

+ +HELP_WGET_OPTIONS_HEADER = the common options used with +HELP_WGET_OPTIONS_TEXT = \ +

\ + For downloading large number of files with wget, the following options might \ + be useful: \ +

\ +
    \ +
  • -t, --tries=NUMBER set number of retries to NUMBER (5 recommended).
    • \ +
  • --auth-no-challenge send Basic HTTP authentication information without \ + waiting for the server's challenge thus saving a \ + roundtrip.
    • \ +
  • --waitretry=SECONDS wait 1..SECONDS between retries of a retrieval. \ + By default, wget will assume a value of 10 seconds.
    • \ +
  • -N, --timestamping Turn on time-stamping and download only missing or \ + updated files.
    • \ +
diff --git a/downloadManager/src/main/resources/ca/nrc/cadc/downloadManager/downloadManagerBundle_fr.properties b/downloadManager/src/main/resources/ca/nrc/cadc/downloadManager/downloadManagerBundle_fr.properties new file mode 100644 index 0000000..4252668 --- /dev/null +++ b/downloadManager/src/main/resources/ca/nrc/cadc/downloadManager/downloadManagerBundle_fr.properties @@ -0,0 +1,97 @@ +# Title and page header +TITLE = Gestionnaire de téléchargement +DOWNLOAD_LINK = /fr/telecharger/ +CAOM_PACKAGE_ENDPOINT = /fr/telecharger/package +PAGE_HEADER = Choisissez une des méthodes de téléchargement suivantes: +DATE_MODIFIED_LABEL = Date de modification + +# Buttons +URL_LIST_BUTTON = List d'URLs dans un fichier +HTML_LIST_BUTTON = List d'URLs sur une page WEB +SHELL_SCRIPT_BUTTON = Accès au script shell +TAR_PACKAGE_BUTTON = Fichier TAR +ZIP_PACKAGE_BUTTON = Fichier ZIP + +# Download type descriptions +JAVA_WEB_START_DESCRIPTION = Le gestionnaire de téléchargement est lancé comme une \ +application de bureau via Java Webstart. Le logiciel est alors automatiquement mis en \ +cache sur votre ordinateur, de sorte que les démarrages subséquants sont plus rapides. + +TAR_PACKAGE_DESCRIPTION = Obtenez un fichier TAR (extension .tar). + +ZIP_PACKAGE_DESCRIPTION = Obtenez un fichier ZIP (extension .zip). + +SHELL_SCRIPT_DESCRIPTION = Obtenez un script Bash exécutable en ligne de commande. Destiné aux utilisateurs á l'aise \ +avec la ligne de commande. Un jeton d'autorisation intégré dans le script pour propager l'authentification initiale \ +sera disponible pour les utilisateurs connectés. Ce jeton expirera aprs un certain délai dés le téléchargement. \ +

\ +$ . ./cadc-download-xxxx.sh + +URL_LIST_DESCRIPTION = Un fichier texte contenant une liste d'URLs est téléchargé. Ce fichier peut par la suite être \ +utilisé avec un script ou directement avec la commande wget. Chaque ligne contient une \ +URL ou, lorsque la génération de l'URL échoue, un message d'erreur. \ +

\ +La commande wget pour télécharger tous les fichiers correspondants aux URLs du fichier téléchargé est la suivante: \ +
\ +% wget --content-disposition -i NOM_DE_FICHIER \ +

\ +Noter que seulement les fichiers publiques sont disponibles si cette option est sélectionnée. +Voir ci-dessous pour plus d'options possibles pour la commande wget: \ +

\ +Noter que seulement les fichiers publiques sont disponibles via cette option. \ +
\ +Voir ci-dessous pour plus d'options wget. + +HTML_LIST_DESCRIPTION = Permet de visualiser la liste des URLs (une par fichier) dans \ +une page Web et de sélectionner individuellement les fichiers à télécharger. + +# Remember download method +REMEMBER_CHECKBOX_LABEL = Retenir mon choix de méthode de téléchargement +REMEMBER_CHECKBOX_TEXT = (cookies requis) +REMEMBER_TEXT = Chaque page de téléchargement possède un bouton \"Choisissez l'une des \ +autres méthodes de téléchargement\" qui, lorsque sélectionné, supprime le choix de \ +téléchargement retenu et retourne vers cette page à choix multiples. + +# Help +HELP_HEADER = Aide + +HELP_WGET_NOT_WORKING_HEADER = ne fonctionne pas +HELP_WGET_NOT_WORKING_TEXT = \ +

\ + L'utilisation recommandée ci-dessus comprend l’option \ + --content-disposition, disponible dans les version 1.12 ou ultérieure \ + de wget. Cette option améliore la probabilité que les fichiers enregistrés \ + le seront avec les noms de fichiers corrects. \ +

\ +

\ + Veuillez noter qu'il existe de nombreuses versions de wget avec une variété \ + d'options et de syntaxe. S'il vous plaît consulter vos pages d'aide locaux. \ +  wget --help devrait afficher les options de votre version de \ + wget. \ +

\ +

\ + La commande wget devrait être disponible sur la plupart des systèmes. \ + Sinon, wget peut être téléchargé à partir de \ + gnu.org. \ + Alternativement, vous pouvez essayer l'un des nombreux autres services de \ + téléchargement tels que: curl, HTTrack, sangsue (Mozilla Add-on), pavuk, lftp, etc. \ +

+ +HELP_WGET_OPTIONS_HEADER = les options couramment utilisées avec +HELP_WGET_OPTIONS_TEXT = \ +

\ + Pour télécharger un grand nombre de fichiers avec wget, les options \ + suivantes pourraient être utiles: \ +

\ +
    \ +
  • -t, --tries=NOMBRE définir le nombre de tentatives à NOMBRE (5 recommandé). \ +
    • \ +
  • --auth-no-challenge envoyer des informations d'authentification Basic HTTP \ + sans attendre l’interrogation du serveur, économisant \ + ainsi un aller-retour.
    • \ +
  • --waitretry=SECONDS attendre 1 à SECONDES entre les tentatives d'une \ + récupération. Par défaut, wget prendra une valeur de \ + 10 secondes.
    • \ +
  • -N, --timestamping Activez l’horodatage et téléchargez uniquement les \ + fichiers manquants ou mises à jour.
    • \ +
diff --git a/downloadManager/src/main/resources/cadcDownload b/downloadManager/src/main/resources/cadcDownload new file mode 100755 index 0000000..38dcda5 --- /dev/null +++ b/downloadManager/src/main/resources/cadcDownload @@ -0,0 +1,216 @@ +#!/bin/bash + +VERBOSE=1 + +usage() +{ + echo "cadcDownload 0.1 (c) Canadian Astronomy Data Centre" + echo + echo "This is a simple download script that uses curl to download a file" + echo "and md5sum to verify the downloaded file. The md5 check is performed" + echo "if the --md5 option is given and the http headers include a" + echo "Content-MD5 value. The script can also read a list of URLs from a text" + echo "file and download all the files." + echo + echo "* Usage: " + echo " cadcDownload [--md5] " + echo " cadcDownload [--md5] " + echo + echo "* Output: " + echo " []" + echo " FAIL " + echo " [] FAIL " + echo + echo "When doing bulk downloads (many files), it is best to redirect the output" + echo "to a file (output described abvoe). To retry the failed downloads, simply" + echo "extract the URLs that FAILED to create a new manifest file, e.g.:" + echo + echo "> cadcDownload --md5 manifest1.txt > output1.log" + echo + echo "## maybe examine output1.log to see what kinds of failures occurred ##" + echo + echo "> grep FAIL output1.log | awk '{print \$1}' > manifest2.txt" + echo + echo "> cadcDownload --md5 manifest2.txt > output2.log" + echo + echo "* Authentication: " + echo + echo "The cadcDownload script calls curl with the --netrc file option. This" + echo "means that if challenged for username and password, curl will read the" + echo "username and password for the server from this file (see: man netrc)." + echo "To download data from CADC (including VOSpace) you will need the following" + echo "servers in your .netrc:" + echo + echo "machine www.cadc-ccda.hia-iha.nrc-cnrc.gc.ca login password " + echo "machine www.canfar.phys.uvic.ca login password " + echo + echo "* Exit status: 0 == OK" + echo " 1 == invalid command-line" + echo " 2 == could not find curl executable" + echo " 3 == server responded with 401 or 403 (authentication)" + echo " 4 == other fatal error" + echo " 5 == MD5 mismatch" + echo " 9 == server responded with a 503 (retry)" + echo +} + +# $1 == URL +doDownload() +{ + URL="$1" + META="tmp-meta.$$" + DATA="tmp-data.$$" + + # Download the file with curl, writing the HTTP header to $META and + # the bytes to $DATA + # --netrc to support username/password auth using $HOME/.netrc (man netrc) + # --silent so we don't put extra stuff to stdout/stderr + # --location to follow any redirects we might get + # --retry=4 for up to 4 retries on transient errors + ACTUAL_MD5="" + declare -i CURL_STATUS=0 + if [ -z $MD5SUM ]; then + # normal download + $CURL --netrc --silent --location $URL -D $META -o $DATA + CURL_STATUS=$? + else + # use tee to send byte stream to file and through md5sum + MD5OUT=$($CURL --netrc --silent --location $URL -D $META | tee $DATA | $MD5SUM) + CURL_STATUS=$? + ACTUAL_MD5=$(echo $MD5OUT | awk '{print $1}') + fi + + ## TODO: check the CURL_STATUS before looking at the headers since failures during + ## download will only show up in the status code + + # use the HTTP headers in $META to determine success/fail + # tail -1 to get the last HTTP status code in case of redirects + # awk -f '[\r]' to strip carriage return + # cut 2- to pick up 2nd and succeeding tokens since message maybe be multiple words + MSG=$(grep '^HTTP/' $META | tail -1 | awk -F '[\r]' '{print $1 $2}' | cut -d' ' -f 2-) + + declare -i STATUS=$(echo $MSG | awk '{print $1}') + + if [ $STATUS != 200 ]; then + EXIT_VALUE=4 + if [ $STATUS == 503 ]; then + echo "$URL FAIL transient error: $MSG" + EXIT_VALUE=9 + elif [ $STATUS == 401 ]; then + echo "$URL FAIL auth error: $MSG" + EXIT_VALUE=3 + elif [ $STATUS == 403 ]; then + echo "$URL FAIL auth error: $MSG" + EXIT_VALUE=3 + else + echo "$URL FAIL error: $MSG" + fi + \rm $META $DATA + return $EXIT_VALUE + fi + + # use the HTTP headers in $META to determine filename + # split on = since the filename is to the right (assuming the whole line is correctly formed) + # awk -F '[\r]' to strip carriage return + FNAME=$(grep '^Content-Disposition' $META | awk -F '=' '{print $2}' | awk -F '[\r]' '{print $1}') + if [ -z $FNAME ]; then + FNAME=$DATA + fi + + ## CADC data web service always sets Content-Disposition so the following is not needed but if + ## you want to use this script with other sites one of these might help... + + # some extra magic to get a filename when Content-Disposition is missing, instead of just using $DATA + # this one keeps the query string and converts special characters into underscores: + #if [ -z $FNAME ]; then + # FNAME=$(echo $URL | awk -F '/' '{print $NF}' | sed 's/[?&]/_/g') + #fi + + # some extra magic to get a filename when Content-Disposition is missing, instead of just using $DATA + # this one drops any query string and uses the base: not that useful if the query string is the only + # differentiating part: + #if [ -z $FNAME ]; then + # FNAME=$(echo $URL | awk -F '/' '{print $NF}' | awk -F '?' '{print $1}') + #fi + + # awk -F '[\r]' to strip carriage return + MD5=$(grep '^Content-MD5' $META | awk '{print $2}' | awk -F '[\r]' '{print $1}') + if [ ! -z $ACTUAL_MD5 ]; then + if [ -z $MD5 ]; then + test $VERBOSE == 1 && echo "no MD5 in http header... skipping check" + else + if [ $MD5 != $ACTUAL_MD5 ]; then + echo "$URL $FNAME $MD5 [FAIL] actual md5: $ACTUAL_MD5" + return 5 + fi + fi + fi + + if [ $DATA != "$FNAME" ]; then + \mv $DATA "$FNAME" + fi + + \rm $META + echo "${URL}" ${FNAME} ${MD5} +} + +## start main part of script + +if [ -z "$1" ]; then + usage + exit 1 +fi + +# find curl executable or fail +CURL=$(which curl) +if [ $? == 0 ]; then + test $VERBOSE == 1 && echo "found: $CURL" +else + echo "not found: curl" + exit 2 +fi +MD5SUM="" + +# process command-line args +INPUT="$1" + +if [ "$1" == "--md5" ]; then + # find md5sum executable or fail + INPUT="$2" + MD5SUM=$(which md5sum) + if [ $? == 0 ]; then + test $VERBOSE == 1 && echo "found: $MD5SUM" + else + echo "not found: md5sum" + exit 2 + fi +fi + +if [ -z "$INPUT" ]; then + usage + exit 1 +fi + +# if input is a file, read URLs from it +if [ -f "$INPUT" ]; then + while read line + do + # awk -F '[\r]' to strip carriage return + URL=$(echo $line | awk -F '[\r]' '{print $1}') + doDownload "$URL" + #EV=$? + #if [ $EV = 9 ]; then + # test -f retry-list.$$ || touch retry-list.$$ + #fi + done < ${INPUT} +else + # assume INPUT is a single URL + doDownload "$INPUT" +fi + + + + + + + diff --git a/downloadManager/src/main/webapp/WEB-INF/c.tld b/downloadManager/src/main/webapp/WEB-INF/c.tld new file mode 100644 index 0000000..22698c9 --- /dev/null +++ b/downloadManager/src/main/webapp/WEB-INF/c.tld @@ -0,0 +1,563 @@ + + + + + JSTL 1.1 core library + JSTL core + 1.1 + c + http://java.sun.com/jsp/jstl/core + + + + Provides core validation features for JSTL tags. + + + org.apache.taglibs.standard.tlv.JstlCoreTLV + + + + + + Catches any Throwable that occurs in its body and optionally + exposes it. + + catch + org.apache.taglibs.standard.tag.common.core.CatchTag + JSP + + +Name of the exported scoped variable for the +exception thrown from a nested action. The type of the +scoped variable is the type of the exception thrown. + + var + false + false + + + + + + Simple conditional tag that establishes a context for + mutually exclusive conditional operations, marked by + <when> and <otherwise> + + choose + org.apache.taglibs.standard.tag.common.core.ChooseTag + JSP + + + + + Simple conditional tag, which evalutes its body if the + supplied condition is true and optionally exposes a Boolean + scripting variable representing the evaluation of this condition + + if + org.apache.taglibs.standard.tag.rt.core.IfTag + JSP + + +The test condition that determines whether or +not the body content should be processed. + + test + true + true + boolean + + + +Name of the exported scoped variable for the +resulting value of the test condition. The type +of the scoped variable is Boolean. + + var + false + false + + + +Scope for var. + + scope + false + false + + + + + + Retrieves an absolute or relative URL and exposes its contents + to either the page, a String in 'var', or a Reader in 'varReader'. + + import + org.apache.taglibs.standard.tag.rt.core.ImportTag + org.apache.taglibs.standard.tei.ImportTEI + JSP + + +The URL of the resource to import. + + url + true + true + + + +Name of the exported scoped variable for the +resource's content. The type of the scoped +variable is String. + + var + false + false + + + +Scope for var. + + scope + false + false + + + +Name of the exported scoped variable for the +resource's content. The type of the scoped +variable is Reader. + + varReader + false + false + + + +Name of the context when accessing a relative +URL resource that belongs to a foreign +context. + + context + false + true + + + +Character encoding of the content at the input +resource. + + charEncoding + false + true + + + + + + The basic iteration tag, accepting many different + collection types and supporting subsetting and other + functionality + + forEach + org.apache.taglibs.standard.tag.rt.core.ForEachTag + org.apache.taglibs.standard.tei.ForEachTEI + JSP + + +Collection of items to iterate over. + + items + false + true + java.lang.Object + + + +If items specified: +Iteration begins at the item located at the +specified index. First item of the collection has +index 0. +If items not specified: +Iteration begins with index set at the value +specified. + + begin + false + true + int + + + +If items specified: +Iteration ends at the item located at the +specified index (inclusive). +If items not specified: +Iteration ends when index reaches the value +specified. + + end + false + true + int + + + +Iteration will only process every step items of +the collection, starting with the first one. + + step + false + true + int + + + +Name of the exported scoped variable for the +current item of the iteration. This scoped +variable has nested visibility. Its type depends +on the object of the underlying collection. + + var + false + false + + + +Name of the exported scoped variable for the +status of the iteration. Object exported is of type +javax.servlet.jsp.jstl.core.LoopTagStatus. This scoped variable has nested +visibility. + + varStatus + false + false + + + + + + Iterates over tokens, separated by the supplied delimeters + + forTokens + org.apache.taglibs.standard.tag.rt.core.ForTokensTag + JSP + + +String of tokens to iterate over. + + items + true + true + java.lang.String + + + +The set of delimiters (the characters that +separate the tokens in the string). + + delims + true + true + java.lang.String + + + +Iteration begins at the token located at the +specified index. First token has index 0. + + begin + false + true + int + + + +Iteration ends at the token located at the +specified index (inclusive). + + end + false + true + int + + + +Iteration will only process every step tokens +of the string, starting with the first one. + + step + false + true + int + + + +Name of the exported scoped variable for the +current item of the iteration. This scoped +variable has nested visibility. + + var + false + false + + + +Name of the exported scoped variable for the +status of the iteration. Object exported is of +type +javax.servlet.jsp.jstl.core.LoopTag +Status. This scoped variable has nested +visibility. + + varStatus + false + false + + + + + + Like <%= ... >, but for expressions. + + out + org.apache.taglibs.standard.tag.rt.core.OutTag + JSP + + +Expression to be evaluated. + + value + true + true + + + +Default value if the resulting value is null. + + default + false + true + + + +Determines whether characters <,>,&,'," in the +resulting string should be converted to their +corresponding character entity codes. Default value is +true. + + escapeXml + false + true + + + + + + + Subtag of <choose> that follows <when> tags + and runs only if all of the prior conditions evaluated to + 'false' + + otherwise + org.apache.taglibs.standard.tag.common.core.OtherwiseTag + JSP + + + + + Adds a parameter to a containing 'import' tag's URL. + + param + org.apache.taglibs.standard.tag.rt.core.ParamTag + JSP + + +Name of the query string parameter. + + name + true + true + + + +Value of the parameter. + + value + false + true + + + + + + Redirects to a new URL. + + redirect + org.apache.taglibs.standard.tag.rt.core.RedirectTag + JSP + + +The URL of the resource to redirect to. + + url + false + true + + + +Name of the context when redirecting to a relative URL +resource that belongs to a foreign context. + + context + false + true + + + + + + Removes a scoped variable (from a particular scope, if specified). + + remove + org.apache.taglibs.standard.tag.common.core.RemoveTag + empty + + +Name of the scoped variable to be removed. + + var + true + false + + + +Scope for var. + + scope + false + false + + + + + + Sets the result of an expression evaluation in a 'scope' + + set + org.apache.taglibs.standard.tag.rt.core.SetTag + JSP + + +Name of the exported scoped variable to hold the value +specified in the action. The type of the scoped variable is +whatever type the value expression evaluates to. + + var + false + false + + + +Expression to be evaluated. + + value + false + true + + + +Target object whose property will be set. Must evaluate to +a JavaBeans object with setter property property, or to a +java.util.Map object. + + target + false + true + + + +Name of the property to be set in the target object. + + property + false + true + + + +Scope for var. + + scope + false + false + + + + + + Creates a URL with optional query parameters. + + url + org.apache.taglibs.standard.tag.rt.core.UrlTag + JSP + + +Name of the exported scoped variable for the +processed url. The type of the scoped variable is +String. + + var + false + false + + + +Scope for var. + + scope + false + false + + + +URL to be processed. + + value + false + true + + + +Name of the context when specifying a relative URL +resource that belongs to a foreign context. + + context + false + true + + + + + + Subtag of <choose> that includes its body if its + condition evalutes to 'true' + + when + org.apache.taglibs.standard.tag.rt.core.WhenTag + JSP + + +The test condition that determines whether or not the +body content should be processed. + + test + true + true + boolean + + + + diff --git a/downloadManager/src/main/webapp/WEB-INF/fmt.tld b/downloadManager/src/main/webapp/WEB-INF/fmt.tld new file mode 100644 index 0000000..5d783e9 --- /dev/null +++ b/downloadManager/src/main/webapp/WEB-INF/fmt.tld @@ -0,0 +1,712 @@ + + + + + + JSTL 1.1 i18n-capable formatting library + JSTL fmt + 1.1 + fmt + http://java.sun.com/jsp/jstl/fmt + + + + Provides core validation features for JSTL tags. + + + org.apache.taglibs.standard.tlv.JstlFmtTLV + + + + + + Sets the request character encoding + + requestEncoding + org.apache.taglibs.standard.tag.rt.fmt.RequestEncodingTag + empty + + +Name of character encoding to be applied when +decoding request parameters. + + value + false + true + + + + + + Stores the given locale in the locale configuration variable + + setLocale + org.apache.taglibs.standard.tag.rt.fmt.SetLocaleTag + empty + + +A String value is interpreted as the +printable representation of a locale, which +must contain a two-letter (lower-case) +language code (as defined by ISO-639), +and may contain a two-letter (upper-case) +country code (as defined by ISO-3166). +Language and country codes must be +separated by hyphen (-) or underscore +(_). + + value + true + true + + + +Vendor- or browser-specific variant. +See the java.util.Locale javadocs for +more information on variants. + + variant + false + true + + + +Scope of the locale configuration variable. + + scope + false + false + + + + + + Specifies the time zone for any time formatting or parsing actions + nested in its body + + timeZone + org.apache.taglibs.standard.tag.rt.fmt.TimeZoneTag + JSP + + +The time zone. A String value is interpreted as +a time zone ID. This may be one of the time zone +IDs supported by the Java platform (such as +"America/Los_Angeles") or a custom time zone +ID (such as "GMT-8"). See +java.util.TimeZone for more information on +supported time zone formats. + + value + true + true + + + + + + Stores the given time zone in the time zone configuration variable + + setTimeZone + org.apache.taglibs.standard.tag.rt.fmt.SetTimeZoneTag + empty + + +The time zone. A String value is interpreted as +a time zone ID. This may be one of the time zone +IDs supported by the Java platform (such as +"America/Los_Angeles") or a custom time zone +ID (such as "GMT-8"). See java.util.TimeZone for +more information on supported time zone +formats. + + value + true + true + + + +Name of the exported scoped variable which +stores the time zone of type +java.util.TimeZone. + + var + false + false + + + +Scope of var or the time zone configuration +variable. + + scope + false + false + + + + + + Loads a resource bundle to be used by its tag body + + bundle + org.apache.taglibs.standard.tag.rt.fmt.BundleTag + JSP + + +Resource bundle base name. This is the bundle's +fully-qualified resource name, which has the same +form as a fully-qualified class name, that is, it uses +"." as the package component separator and does not +have any file type (such as ".class" or ".properties") +suffix. + + basename + true + true + + + +Prefix to be prepended to the value of the message +key of any nested <fmt:message> action. + + prefix + false + true + + + + + + Loads a resource bundle and stores it in the named scoped variable or + the bundle configuration variable + + setBundle + org.apache.taglibs.standard.tag.rt.fmt.SetBundleTag + empty + + +Resource bundle base name. This is the bundle's +fully-qualified resource name, which has the same +form as a fully-qualified class name, that is, it uses +"." as the package component separator and does not +have any file type (such as ".class" or ".properties") +suffix. + + basename + true + true + + + +Name of the exported scoped variable which stores +the i18n localization context of type +javax.servlet.jsp.jstl.fmt.LocalizationC +ontext. + + var + false + false + + + +Scope of var or the localization context +configuration variable. + + scope + false + false + + + + + + Maps key to localized message and performs parametric replacement + + message + org.apache.taglibs.standard.tag.rt.fmt.MessageTag + JSP + + +Message key to be looked up. + + key + false + true + + + +Localization context in whose resource +bundle the message key is looked up. + + bundle + false + true + + + +Name of the exported scoped variable +which stores the localized message. + + var + false + false + + + +Scope of var. + + scope + false + false + + + + + + Supplies an argument for parametric replacement to a containing + <message> tag + + param + org.apache.taglibs.standard.tag.rt.fmt.ParamTag + JSP + + +Argument used for parametric replacement. + + value + false + true + + + + + + Formats a numeric value as a number, currency, or percentage + + formatNumber + org.apache.taglibs.standard.tag.rt.fmt.FormatNumberTag + JSP + + +Numeric value to be formatted. + + value + false + true + + + +Specifies whether the value is to be +formatted as number, currency, or +percentage. + + type + false + true + + + +Custom formatting pattern. + + pattern + false + true + + + +ISO 4217 currency code. Applied only +when formatting currencies (i.e. if type is +equal to "currency"); ignored otherwise. + + currencyCode + false + true + + + +Currency symbol. Applied only when +formatting currencies (i.e. if type is equal +to "currency"); ignored otherwise. + + currencySymbol + false + true + + + +Specifies whether the formatted output +will contain any grouping separators. + + groupingUsed + false + true + + + +Maximum number of digits in the integer +portion of the formatted output. + + maxIntegerDigits + false + true + + + +Minimum number of digits in the integer +portion of the formatted output. + + minIntegerDigits + false + true + + + +Maximum number of digits in the +fractional portion of the formatted output. + + maxFractionDigits + false + true + + + +Minimum number of digits in the +fractional portion of the formatted output. + + minFractionDigits + false + true + + + +Name of the exported scoped variable +which stores the formatted result as a +String. + + var + false + false + + + +Scope of var. + + scope + false + false + + + + + + Parses the string representation of a number, currency, or percentage + + parseNumber + org.apache.taglibs.standard.tag.rt.fmt.ParseNumberTag + JSP + + +String to be parsed. + + value + false + true + + + +Specifies whether the string in the value +attribute should be parsed as a number, +currency, or percentage. + + type + false + true + + + +Custom formatting pattern that determines +how the string in the value attribute is to be +parsed. + + pattern + false + true + + + +Locale whose default formatting pattern (for +numbers, currencies, or percentages, +respectively) is to be used during the parse +operation, or to which the pattern specified +via the pattern attribute (if present) is +applied. + + parseLocale + false + true + + + +Specifies whether just the integer portion of +the given value should be parsed. + + integerOnly + false + true + + + +Name of the exported scoped variable which +stores the parsed result (of type +java.lang.Number). + + var + false + false + + + +Scope of var. + + scope + false + false + + + + + + Formats a date and/or time using the supplied styles and pattern + + formatDate + org.apache.taglibs.standard.tag.rt.fmt.FormatDateTag + empty + + +Date and/or time to be formatted. + + value + true + true + + + +Specifies whether the time, the date, or both +the time and date components of the given +date are to be formatted. + + type + false + true + + + +Predefined formatting style for dates. Follows +the semantics defined in class +java.text.DateFormat. Applied only +when formatting a date or both a date and +time (i.e. if type is missing or is equal to +"date" or "both"); ignored otherwise. + + dateStyle + false + true + + + +Predefined formatting style for times. Follows +the semantics defined in class +java.text.DateFormat. Applied only +when formatting a time or both a date and +time (i.e. if type is equal to "time" or "both"); +ignored otherwise. + + timeStyle + false + true + + + +Custom formatting style for dates and times. + + pattern + false + true + + + +Time zone in which to represent the formatted +time. + + timeZone + false + true + + + +Name of the exported scoped variable which +stores the formatted result as a String. + + var + false + false + + + +Scope of var. + + scope + false + false + + + + + + Parses the string representation of a date and/or time + + parseDate + org.apache.taglibs.standard.tag.rt.fmt.ParseDateTag + JSP + + +Date string to be parsed. + + value + false + true + + + +Specifies whether the date string in the +value attribute is supposed to contain a +time, a date, or both. + + type + false + true + + + +Predefined formatting style for days +which determines how the date +component of the date string is to be +parsed. Applied only when formatting a +date or both a date and time (i.e. if type +is missing or is equal to "date" or "both"); +ignored otherwise. + + dateStyle + false + true + + + +Predefined formatting styles for times +which determines how the time +component in the date string is to be +parsed. Applied only when formatting a +time or both a date and time (i.e. if type +is equal to "time" or "both"); ignored +otherwise. + + timeStyle + false + true + + + +Custom formatting pattern which +determines how the date string is to be +parsed. + + pattern + false + true + + + +Time zone in which to interpret any time +information in the date string. + + timeZone + false + true + + + +Locale whose predefined formatting styles +for dates and times are to be used during +the parse operation, or to which the +pattern specified via the pattern +attribute (if present) is applied. + + parseLocale + false + true + + + +Name of the exported scoped variable in +which the parsing result (of type +java.util.Date) is stored. + + var + false + false + + + +Scope of var. + + scope + false + false + + + + diff --git a/downloadManager/src/main/webapp/WEB-INF/web.xml b/downloadManager/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 0000000..1f7f4ac --- /dev/null +++ b/downloadManager/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,109 @@ + + + downloadManager + + + logControl + ca.nrc.cadc.log.LogControlServlet + + logLevel + info + + + logLevelPackages + + ca.nrc.cadc.dali + ca.nrc.cadc.tap + org.opencadc.alma + org.opencadc.tap + ca.nrc.cadc.rest + ca.nrc.cadc.uws + + + + logControlProperties + dlm-logControl.properties + + 1 + + + + + download + ca.nrc.cadc.dlm.server.DispatcherServlet + 2 + + + + JavaWebstart + ca.nrc.cadc.dlm.server.JavaWebStartServlet + + + + TARPackageServlet + ca.nrc.cadc.dlm.server.TARPackageServlet + + + + ZIPPackageServlet + ca.nrc.cadc.dlm.server.ZIPPackageServlet + + + + urlList + ca.nrc.cadc.dlm.server.UrlListServlet + + + + shellScript + ca.nrc.cadc.dlm.server.ShellScriptServlet + + + + download + / + + + + JavaWebstart + /javaWebstart + + + + JavaWebstart + /DownloadManager.jnlp + + + default + /codebase/* + + + + TARPackageServlet + /tar-package + + + + ZIPPackageServlet + /zip-package + + + + urlList + /urlList + + + + shellScript + /shellScript + + + + logControl + /logControl + + + diff --git a/downloadManager/test/src/resources/TestPage.html b/downloadManager/test/src/resources/TestPage.html new file mode 100644 index 0000000..6c3ea7e --- /dev/null +++ b/downloadManager/test/src/resources/TestPage.html @@ -0,0 +1,184 @@ + + + + + + + + +

DataLink Service: caom scheme

+ + +
+ + +
+ +
+ + + +
+ +
+ + + + + +
+ +
+ + + +
+ +
+ + + + +
+ +
+ + + +
+ +
+ + + +
+ +
+ + + + + +
+ +

VOSpace manifest tests

+ +
+ + +
+ +
+ + +
+ +
+ + +
+ +

OLA en/fr tests

+ +
+ + +
+ +
+ + +
+ +

Data web service: ad scheme

+ +
+ + +
+ +
+ + + + + + + +
+ +
+ + + + +
+ +
+ + + + +
+ +
+ + + + + + + + +
+ +
+ + + + + +
+ +
+ + +
+ +
+ + + +
+ + + + +
+ + +
+ + +

miscellaneous tests

+ +
+ + + +
+ +
+ + + +
+ +
+ +
+ + + diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ec991f9..3ae1e2f 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-6.9.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/opencadc.gradle b/opencadc.gradle index 1f83014..2f59499 100644 --- a/opencadc.gradle +++ b/opencadc.gradle @@ -1,10 +1,14 @@ configurations { checkstyleDep + intTestImplementation.extendsFrom testImplementation + + runtimeOnly.exclude module: 'slf4j-reload4j' + runtimeOnly.exclude module: 'reload4j' } dependencies { testImplementation 'com.puppycrawl.tools:checkstyle:8.2' - checkstyleDep 'org.opencadc:cadc-quality:1.+' + checkstyleDep 'org.opencadc:cadc-quality:[1.0,)' } checkstyle { @@ -14,16 +18,70 @@ checkstyle { sourceSets = [] } -// Temporary work around for issue https://github.com/gradle/gradle/issues/881 - +sourceSets { + intTest { + java { + compileClasspath += main.output + test.output + runtimeClasspath += compileClasspath + } + resources.srcDirs += file('src/intTest/resources') + resources.srcDirs += new File(System.getenv('A') + '/test-certificates/') + } +} + +// Temporary work around for issue https://github.com/gradle/gradle/issues/881 - // gradle not displaying fail build status when warnings reported --> +tasks.withType(Checkstyle).configureEach { + reports { + xml.required.set(false) + html.required.set(true) + } +} + +tasks.withType(Test).configureEach { + // reset the report destinations so that intTests go to their own page + reports.html.destination = file(reporting.baseDir.getAbsolutePath() + '/' + name) + + // Assign all Java system properties from + // the command line to the tests + systemProperties System.properties +} + +tasks.register('intTest', Test) { + // set the configuration context + testClassesDirs = sourceSets.intTest.output.classesDirs + classpath = sourceSets.intTest.runtimeClasspath -tasks.withType(Checkstyle).each { checkstyleTask -> - checkstyleTask.doLast { - reports.all { report -> - def outputFile = report.destination - if (outputFile.exists() && outputFile.text.contains(" Date: Mon, 20 Oct 2025 15:31:53 -0700 Subject: [PATCH 2/3] fix: fix cadc download manager tests --- .../java/ca/nrc/cadc/dlm/DownloadTuple.java | 6 ++-- .../handlers/CadcDownloadGeneratorTest.java | 32 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/cadc-download-manager/src/main/java/ca/nrc/cadc/dlm/DownloadTuple.java b/cadc-download-manager/src/main/java/ca/nrc/cadc/dlm/DownloadTuple.java index 0c4a352..828addf 100644 --- a/cadc-download-manager/src/main/java/ca/nrc/cadc/dlm/DownloadTuple.java +++ b/cadc-download-manager/src/main/java/ca/nrc/cadc/dlm/DownloadTuple.java @@ -68,7 +68,7 @@ package ca.nrc.cadc.dlm; -import ca.nrc.cadc.dali.DoubleInterval; +import ca.nrc.cadc.dali.Interval; import ca.nrc.cadc.dali.Shape; import java.net.URI; @@ -79,7 +79,7 @@ public class DownloadTuple { // These values can all be null public Shape posCutout; - public DoubleInterval bandCutout; + public Interval bandCutout; public String pixelCutout; public String label; @@ -106,7 +106,7 @@ public DownloadTuple(URI id) { * @param pixelCutout (Optional) String to be used for pixel cutout (passed through without validation) * @param label (Optional) sent as LABEL parameter to SODA calls */ - public DownloadTuple(URI id, Shape posCutout, DoubleInterval bandCutout, String pixelCutout, String label) { + public DownloadTuple(URI id, Shape posCutout, Interval bandCutout, String pixelCutout, String label) { if (id == null) { throw new IllegalArgumentException("id can not be null"); } diff --git a/cadc-download-manager/src/test/java/ca/nrc/cadc/dlm/handlers/CadcDownloadGeneratorTest.java b/cadc-download-manager/src/test/java/ca/nrc/cadc/dlm/handlers/CadcDownloadGeneratorTest.java index 4b14429..dfa9c8d 100644 --- a/cadc-download-manager/src/test/java/ca/nrc/cadc/dlm/handlers/CadcDownloadGeneratorTest.java +++ b/cadc-download-manager/src/test/java/ca/nrc/cadc/dlm/handlers/CadcDownloadGeneratorTest.java @@ -32,6 +32,7 @@ import ca.nrc.cadc.caom2.artifact.resolvers.CadcResolver; import ca.nrc.cadc.dali.Circle; import ca.nrc.cadc.dali.DoubleInterval; +import ca.nrc.cadc.dali.Interval; import ca.nrc.cadc.dali.Point; import ca.nrc.cadc.dali.Polygon; import ca.nrc.cadc.dlm.DownloadDescriptor; @@ -41,13 +42,16 @@ import ca.nrc.cadc.util.Log4jInit; import org.apache.log4j.Level; import org.apache.log4j.Logger; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Iterator; /** @@ -59,11 +63,19 @@ public class CadcDownloadGeneratorTest { Log4jInit.setLevel("ca.nrc.cadc.dlm.handlers", Level.DEBUG); } - URL baseURL; + @Before + public void setUp() { + System.setProperty(RegistryClient.class.getName() + ".host", "ws.cadc-ccda.hia-iha.nrc-cnrc.gc.ca"); + } + + @After + public void tearDown() { + System.clearProperty(RegistryClient.class.getName() + ".host"); + } - public CadcDownloadGeneratorTest() { + private static URL getBaseURL() { final RegistryClient rc = new RegistryClient(); - this.baseURL = rc.getServiceURL(CadcResolver.STORAGE_INVENTORY_URI, Standards.SI_FILES, AuthMethod.ANON); + return rc.getServiceURL(CadcResolver.STORAGE_INVENTORY_URI, Standards.SI_FILES, AuthMethod.ANON); } @Test @@ -78,6 +90,7 @@ public void testURL() { Assert.assertTrue(descriptorIterator.hasNext()); final DownloadDescriptor dd = descriptorIterator.next(); + final URL baseURL = CadcDownloadGeneratorTest.getBaseURL(); Assert.assertEquals("uri", uri.toASCIIString(), dd.uri); Assert.assertEquals("protocol", baseURL.getProtocol(), dd.url.getProtocol()); @@ -110,6 +123,7 @@ public void testURLPlusSUBCutout() { log.debug(dd.status); Assert.assertEquals(DownloadDescriptor.OK, dd.status); + final URL baseURL = CadcDownloadGeneratorTest.getBaseURL(); Assert.assertEquals("uri", uri.toASCIIString(), dd.uri); Assert.assertEquals("protocol", baseURL.getProtocol(), dd.url.getProtocol()); Assert.assertEquals("hostname", baseURL.getHost(), dd.url.getHost()); @@ -141,6 +155,8 @@ public void testURLPlusCIRCLECutout() { log.debug(dd.status); Assert.assertEquals(DownloadDescriptor.OK, dd.status); + final URL baseURL = CadcDownloadGeneratorTest.getBaseURL(); + Assert.assertEquals("uri", uri.toASCIIString(), dd.uri); Assert.assertEquals("protocol", baseURL.getProtocol(), dd.url.getProtocol()); Assert.assertEquals("hostname", baseURL.getHost(), dd.url.getHost()); @@ -162,7 +178,7 @@ public void testURLPlusBANDAndPOLYGONCutout() { final URI uri = URI.create("cadc:archiveName/file_1.fits"); final DownloadTuple dt = new DownloadTuple(uri); - dt.bandCutout = new DoubleInterval(14.6D, 64.1D); + dt.bandCutout = new Interval<>(14.6D, 64.1D); final Polygon polygon = new Polygon(); polygon.getVertices().add(new Point(10.0D, 10.0D)); @@ -180,6 +196,8 @@ public void testURLPlusBANDAndPOLYGONCutout() { log.debug(dd.status); Assert.assertEquals(DownloadDescriptor.OK, dd.status); + final URL baseURL = CadcDownloadGeneratorTest.getBaseURL(); + Assert.assertEquals("uri", uri.toASCIIString(), dd.uri); Assert.assertEquals("protocol", baseURL.getProtocol(), dd.url.getProtocol()); Assert.assertEquals("hostname", baseURL.getHost(), dd.url.getHost()); @@ -219,10 +237,6 @@ public void testIllegalScheme() { } private static String encodeString(String str) { - try { - return URLEncoder.encode(str, "UTF-8"); - } catch (UnsupportedEncodingException ignore) { - } - return null; + return URLEncoder.encode(str, StandardCharsets.UTF_8); } } From 83f103583a95cb3650dc23c64e39ee182b6f88d2 Mon Sep 17 00:00:00 2001 From: Dustin Jenkins Date: Mon, 20 Oct 2025 15:36:13 -0700 Subject: [PATCH 3/3] test: fix existing tests --- cadc-download-manager-server/build.gradle | 2 +- .../main/java/ca/nrc/cadc/dlm/server/ShellScriptServlet.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cadc-download-manager-server/build.gradle b/cadc-download-manager-server/build.gradle index 3e9c7a8..0ad844e 100644 --- a/cadc-download-manager-server/build.gradle +++ b/cadc-download-manager-server/build.gradle @@ -24,7 +24,7 @@ def git_url = 'https://github.com/opencadc/apps' dependencies { implementation 'javax.servlet:javax.servlet-api:[3.1,)' - implementation 'org.opencadc:cadc-app-kit:[1.0,)' + implementation 'org.opencadc:cadc-cdp:[1.4.0,2.0.0)' implementation 'org.opencadc:cadc-dali:[1.2.15,)' implementation 'org.opencadc:cadc-download-manager:[1.5.0,)' implementation 'org.opencadc:cadc-log:[1.0,)' diff --git a/cadc-download-manager-server/src/main/java/ca/nrc/cadc/dlm/server/ShellScriptServlet.java b/cadc-download-manager-server/src/main/java/ca/nrc/cadc/dlm/server/ShellScriptServlet.java index 2e6e875..adcebff 100644 --- a/cadc-download-manager-server/src/main/java/ca/nrc/cadc/dlm/server/ShellScriptServlet.java +++ b/cadc-download-manager-server/src/main/java/ca/nrc/cadc/dlm/server/ShellScriptServlet.java @@ -165,7 +165,7 @@ protected void doPost(HttpServletRequest request, HttpServletResponse response) private String getToken(final Subject subject) throws Exception { final LocalAuthority localAuthority = new LocalAuthority(); - final URI oAuthServiceURI = localAuthority.getServiceURI(Standards.SECURITY_METHOD_OPENID.toString()); + final URI oAuthServiceURI = localAuthority.getResourceID(Standards.SECURITY_METHOD_OPENID); final RegistryClient registryClient = new RegistryClient(); final URL baseOAuthServiceURL = registryClient.getServiceURL(oAuthServiceURI, Standards.SECURITY_METHOD_OPENID, AuthMethod.COOKIE);