From 6176f955d7dffea85ee57d08b855b50dd4061d03 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 1 Jan 2025 16:48:07 -0500 Subject: [PATCH 1/3] E2E start This stands up a service on the local stack that serves pages, and drives the process front-to-back so we can test the whole service. Ultimately, we want this to be something we run against the `staging` stack, but for now... this is better than nothing. --- Makefile | 2 + cmd/e2e/Makefile | 8 + cmd/e2e/README.md | 9 + cmd/e2e/get-page.py | 21 ++ cmd/e2e/main.go | 31 ++ cmd/e2e/run.sh | 3 + cmd/e2e/static/fac-1218.html | 443 ++++++++++++++++++++++++++++ cmd/e2e/static/helio.html | 170 +++++++++++ compose.yaml | 49 ++- config/domain64/README.md | 10 + config/domain64/gov.libsonnet | 7 + config/domain64/schedules.libsonnet | 1 + config/hosts/allowed_hosts.jsonnet | 12 + 13 files changed, 765 insertions(+), 1 deletion(-) create mode 100644 cmd/e2e/Makefile create mode 100644 cmd/e2e/README.md create mode 100644 cmd/e2e/get-page.py create mode 100644 cmd/e2e/main.go create mode 100755 cmd/e2e/run.sh create mode 100644 cmd/e2e/static/fac-1218.html create mode 100644 cmd/e2e/static/helio.html create mode 100644 config/domain64/README.md diff --git a/Makefile b/Makefile index eac6c9e8..387a992c 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,8 @@ build: clean config generate cd cmd/migrate ; make build echo "build admin" cd cmd/admin ; make build + echo "build e2e" + cd cmd/e2e ; make build echo "build entree" cd cmd/entree ; make build echo "build extract" diff --git a/cmd/e2e/Makefile b/cmd/e2e/Makefile new file mode 100644 index 00000000..84be2f9e --- /dev/null +++ b/cmd/e2e/Makefile @@ -0,0 +1,8 @@ +build: + go build -buildvcs=false -o service.exe + +run: + go run *.go + +clean: + rm -f service.exe diff --git a/cmd/e2e/README.md b/cmd/e2e/README.md new file mode 100644 index 00000000..0fdeeaf9 --- /dev/null +++ b/cmd/e2e/README.md @@ -0,0 +1,9 @@ +# get-page + +``` +python get-page.py "URL" page_name +``` + +will pull down the plain text of a page, and wrap it in an HTML body. It will save the page into the `static` folder, adding a `.html` to the end. + +This is to create pages for testing the search engine; only pull CC0/public domain text. \ No newline at end of file diff --git a/cmd/e2e/get-page.py b/cmd/e2e/get-page.py new file mode 100644 index 00000000..2f6a8fcc --- /dev/null +++ b/cmd/e2e/get-page.py @@ -0,0 +1,21 @@ +import bs4 +import click +import requests + +@click.command() +@click.argument('url') +@click.argument('page_name') +def main(url, page_name): + r = requests.get(url) + url_contents = r.text + soup = bs4.BeautifulSoup(url_contents, features="lxml") + contents = soup.getText() + with open(f"static/{page_name}.html", 'w') as fp: + fp.write("\n") + fp.write(f"{page_name}\n") + fp.write("\n") + fp.write(contents.strip()) + fp.write("\n\n") + +if __name__ in "__main__": + main() \ No newline at end of file diff --git a/cmd/e2e/main.go b/cmd/e2e/main.go new file mode 100644 index 00000000..8fcf206f --- /dev/null +++ b/cmd/e2e/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "net/http" + + "github.com/GSA-TTS/jemison/internal/common" + "github.com/GSA-TTS/jemison/internal/env" + "github.com/GSA-TTS/jemison/internal/postgres" + "github.com/GSA-TTS/jemison/internal/queueing" + "go.uber.org/zap" +) + +var ThisServiceName = "e2e" +var ChQSHP = make(chan queueing.QSHP) + +var JDB *postgres.JemisonDB + +func main() { + env.InitGlobalEnv(ThisServiceName) + + engine := common.InitializeAPI() + + go queueing.Enqueue(ChQSHP) + + zap.L().Info("listening to the music of the spheres", + zap.String("port", env.Env.Port)) + + // Serve the static directory at /static + engine.Static("/static", "./static") + http.ListenAndServe(":"+env.Env.Port, engine) +} diff --git a/cmd/e2e/run.sh b/cmd/e2e/run.sh new file mode 100755 index 00000000..c973fb2f --- /dev/null +++ b/cmd/e2e/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +/home/vcap/app/cmd/e2e/service.exe || exit 1 diff --git a/cmd/e2e/static/fac-1218.html b/cmd/e2e/static/fac-1218.html new file mode 100644 index 00000000..d253ef49 --- /dev/null +++ b/cmd/e2e/static/fac-1218.html @@ -0,0 +1,443 @@ + +fac-1218.html + +Updates from the FAC + + + + + + + + + + + + + An official website of the United States government + + + Here’s how you know + + + +Here’s how you know + + + + + + + + + + Official websites use .gov + + A .gov website belongs to an official government + organization in the United States. + + + + + + + + Secure .gov websites use HTTPS + + A lock ( + +Lock +A locked padlock + + ) or https:// means you’ve safely connected to + the .gov website. Share sensitive information only on official, + secure websites. + + + + + + + + + + + + + + +Federal Audit +Clearinghouse + + + + +Menu + + + + + + + + + + +Audit search + + + +Search + + +Search resources + + +Developer resources + + +Data reliability + + +Tribal audits + + + + + +Audit submission + + + +Submission resources + + +Submission home + + + + + +Updates & News + + + +FAC updates + + +OMB announcements + + +System status + + + + + +Policy & Compliance + + + +Compliance + + +Uniform guidance + + +Burden statement + + + + + +Contact + + + +OMB# 3090-0330 EXP: 09/30/2026 + + + + + + + + + Search + + + + + + + + + + + + +Sign in + + + + +You must log in to continue + + + Submitting information to the Federal Audit Clearinghouse requires authentication + which will now be handled by Login.gov. + You cannot use your old Census FAC credentials to access the new GSA + system. On the next screen you can use an existing Login.gov + account or create a new one to securely sign in. + + + + + + +Authenticate with Login.gov + + + +Cancel + + + + + + + + + + + + + + + + + + + +Home + + + +FAC Updates + + + +December 2023 + + + + + + December 2023 + + +Audit access management, Search improvements, Summary reports +December 18, 2023 +This week we've made improvements to user access roles, search results, and Tribal data access. +What we delivered +We work in an agile manner. That means we have a long-term strategy, medium-term features we work to deliver, and make continuous improvement and bug fixes to the existing product. +Audit access management +We improved the user access experience for single audits. Users can now change both the auditor and auditee certifying official for a submission and add new audit editors. +Future improvements will allow for the removal of other contributors to a submission. +Search improvements +We added the ability to filter search results by auditee state. We also enhanced the results returned by the ALN filter, making it possible to see if a submission includes audit findings. +Tribal data access controls +We have implemented access controls for Tribal audit data, and notified NSACs and KMSALs. This data will now appear in searches for logged in Federal users who have access. +Summary reports +We added the ability to download a summary report for an individual single audit submission. This combines all data from a single SF-SAC into one Excel document. +Pre-certification review +Auditors and auditees can now review a submission after it is validated and locked, but before it is certified. +What's next? +Historic data migration +We continue to work with Census on the migration of historic data. + + + + Most Recent Updates + + + + + + August 2024 + + +Historical data migration, Submission exceptions, Fixes and improvements +August 21, 2024 + + + + + + June 2024 + + +Security testing, Historical data migration, Fixes and improvements +June 11, 2024 + + + + + + May 2024 + + +Resubmission, Historical data migration, May code freeze +May 13, 2024 + + + + +View All Updates + + + + + + + Subscribe to stay updated + + + Receive FAC announcements and other Federal grants updates. + + + + + + + + + + + + + + + Grantees and auditors + + + +grantscommunity-subscribe-request@listserv.gsa.gov + + + + + + + + + + + + + + + + Federal awarding agencies + + + (Federal Employees only) + + + +grantsfed-subscribe-request@listserv.gsa.gov + + + + + + + To subscribe, email the relevant address above with the subject line + + "Subscribe" + + to get tailored updates in your inbox. + + + + + + How we work + + + + +The Federal Audit Clearinghouse team works in the open. Our day-to-day task board can be found on Github and prior updates are available in our archive. + + + +View our progress + + + + + + + + + + + + + + + + +FAC.gov + + An official website of the + + GSA’s Technology Transformation Services + + + + + + + + + + + Report fraud, waste, or abuse to the Office of the Inspector General + + + + + Submit a Freedom of Information Act (FOIA) request + + + + + View budget and performance reports + + + + + View accessibility statement + + + + + View No FEAR Act data + + + + + View privacy policy + + + + + + + + + Looking for U.S. government information and services? + +Visit USA.gov + diff --git a/cmd/e2e/static/helio.html b/cmd/e2e/static/helio.html new file mode 100644 index 00000000..5353a9d1 --- /dev/null +++ b/cmd/e2e/static/helio.html @@ -0,0 +1,170 @@ + +helio.html + +NASA’s Parker Solar Probe Makes History With Closest Pass to Sun - NASA Science + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Skip to main content Missions Search All NASA MissionsA to Z List of MissionsUpcoming Launches and LandingsSpaceships and RocketsCommunicating with MissionsArtemisJames Webb Space TelescopeHubble Space TelescopeInternational Space StationOSIRIS-RExHumans in Space Why Go to SpaceAstronautsCommercial SpaceDestinationsSpaceships and RocketsLiving in SpaceEarth & Climate Explore Earth ScienceClimate ChangeEarth, Our PlanetEarth Science in ActionEarth MultimediaEarth DataEarth Science ResearchersAbout NASA NASA's ImpactsCenters and FacilitiesDirectoratesOrganizationsPeople of NASACareersInternshipsOur HistoryDoing Business with NASAGet InvolvedContactLearning Resources For Kids and StudentsFor EducatorsFor Colleges and UniversitiesFor ProfessionalsScience for EveryoneRequest an Exhibit or SpeakerSTEM Engagement at NASAAeronautics Science in the AirNASA AircraftFlight InnovationSupersonic FlightAir Traffic SolutionsGreen Aviation TechDrones & YouTechnology Technology Transfer & SpinoffsSpace Travel TechnologyTechnology Living in SpaceManufacturing and MaterialsRoboticsScience InstrumentsComputingThe Universe ExoplanetsThe Search for Life in the UniverseStarsGalaxiesBlack HolesThe Big BangDark Matter & Dark EnergyThe Solar System The SunMercuryVenusEarthThe MoonMarsJupiterSaturnUranusNeptunePluto & Dwarf PlanetsAsteroids, Comets & MeteorsThe Kuiper BeltThe Oort CloudSkywatchingEspañol CienciaAeronáuticaCiencias TerrestresSistema SolarUniversoScience All NASA ScienceEarth SciencePlanetary ScienceAstrophysics & Space ScienceThe Sun & HeliophysicsBiological & Physical SciencesLunar ScienceCitizen ScienceAstromaterialsAeronautics ResearchHuman Space Travel ResearchExploreSearchSubmitNews & EventsMultimediaNASA+5 min readNASA’s Parker Solar Probe Makes History With Closest Pass to SunAn artist's concept showing Parker Solar Probe.Credits: NASA/APLMara Johnson-GrohDec 27, 2024ArticleOperations teams have confirmed NASA’s mission to “touch” the Sun survived its record-breaking closest approach to the solar surface on Dec. 24, 2024.Breaking its previous record by flying just 3.8 million miles above the surface of the Sun, NASA’s Parker Solar Probe hurtled through the solar atmosphere at a blazing 430,000 miles per hour — faster than any human-made object has ever moved. A beacon tone received late on Dec. 26 confirmed the spacecraft had made it through the encounter safely and is operating normally.This pass, the first of more to come at this distance, allows the spacecraft to conduct unrivaled scientific measurements with the potential to change our understanding of the Sun.Flying this close to the Sun is a historic moment in humanity’s first mission to a star.Nicky foxNASA Associate Administrator, Science Mission Directorate"Flying this close to the Sun is a historic moment in humanity’s first mission to a star,” said Nicky Fox, who leads the Science Mission Directorate at NASA Headquarters in Washington. “By studying the Sun up close, we can better understand its impacts throughout our solar system, including on the technology we use daily on Earth and in space, as well as learn about the workings of stars across the universe to aid in our search for habitable worlds beyond our home planet.”NASA's Parker Solar Probe survived its record-breaking closest approach to the solar surface on Dec. 24, 2024. Breaking its previous record by flying just 3.8 million miles above the surface of the Sun, the spacecraft hurtled through the solar atmosphere at a blazing 430,000 miles per hour — faster than any human-made object has ever moved.Credits: NASAThis video can be freely shared and downloaded at https://svs.gsfc.nasa.gov/14741.Parker Solar Probe has spent the last six years setting up for this moment. Launched in 2018, the spacecraft used seven flybys of Venus to gravitationally direct it ever closer to the Sun. With its last Venus flyby on Nov. 6, 2024, the spacecraft reached its optimal orbit. This oval-shaped orbit brings the spacecraft an ideal distance from the Sun every three months — close enough to study our Sun’s mysterious processes but not too close to become overwhelmed by the Sun’s heat and damaging radiation. The spacecraft will remain in this orbit for the remainder of its primary mission.“Parker Solar Probe is braving one of the most extreme environments in space and exceeding all expectations,” said Nour Rawafi, the project scientist for Parker Solar Probe at the Johns Hopkins Applied Physics Laboratory (APL), which designed, built, and operates the spacecraft from its campus in Laurel, Maryland. “This mission is ushering a new golden era of space exploration, bringing us closer than ever to unlocking the Sun’s deepest and most enduring mysteries.”Close to the Sun, the spacecraft relies on a carbon foam shield to protect it from the extreme heat in the upper solar atmosphere called the corona, which can exceed 1 million degrees Fahrenheit. The shield was designed to reach temperatures of 2,600 degrees Fahrenheit — hot enough to melt steel — while keeping the instruments behind it shaded at a comfortable room temperature. In the hot but low-density corona, the spacecraft’s shield is expected to warm to 1,800 degrees Fahrenheit.The spacecraft’s record close distance of 3.8 million miles may sound far, but on cosmic scales it’s incredibly close. If the solar system was scaled down with the distance between the Sun and Earth the length of a football field, Parker Solar Probe would be just four yards from the end zone — close enough to pass within the tenuous outer atmosphere of the Sun known as the corona.NASA/APL“It’s monumental to be able to get a spacecraft this close to the Sun,” said John Wirzburger, the Parker Solar Probe mission systems engineer at APL. “This is a challenge the space science community has wanted to tackle since 1958 and had spent decades advancing the technology to make it possible.”By flying through the solar corona, Parker Solar Probe can take measurements that help scientists better understand how the region gets so hot, trace the origin of the solar wind (a constant flow of material escaping the Sun), and discover how energetic particles are accelerated to half the speed of light.“The data is so important for the science community because it gives us another vantage point,” said Kelly Korreck, a program scientist at NASA Headquarters and heliophysicist who worked on one of the mission’s instruments. “By getting firsthand accounts of what’s happening in the solar atmosphere, Parker Solar Probe has revolutionized our understanding of the Sun.”Previous passes have already aided scientists’ understanding of the Sun. When the spacecraft first passed into the solar atmosphere in 2021, it found the outer boundary of the corona is wrinkled with spikes and valleys, contrary to what was expected. Parker Solar Probe also pinpointed the origin of important zig-zag-shaped structures in the solar wind, called switchbacks, at the visible surface of the Sun — the photosphere.Since that initial pass into the Sun, the spacecraft has been spending more time in the corona, where most of the critical physical processes occur.This conceptual image shows Parker Solar Probe about to enter the solar corona.NASA/Johns Hopkins APL/Ben Smith“We now understand the solar wind and its acceleration away from the Sun,” said Adam Szabo, the Parker Solar Probe mission scientist at NASA’s Goddard Space Flight Center in Greenbelt, Maryland. “This close approach will give us more data to understand how it’s accelerated closer in.”Parker Solar Probe has also made discoveries across the inner solar system. Observations showed how giant solar explosions called coronal mass ejections vacuum up dust as they sweep across the solar system, and other observations revealed unexpected findings about solar energetic particles. Flybys of Venus have documented the planet’s natural radio emissions from its atmosphere, as well as the first complete image of its orbital dust ring.So far, the spacecraft has only transmitted that it’s safe, but soon it will be in a location that will allow it to downlink the data it collected on this latest solar pass.The data that will come down from the spacecraft will be fresh information about a place that we, as humanity, have never been.Joe WestlakeHeliophysics Division Director, NASA Headquarters“The data that will come down from the spacecraft will be fresh information about a place that we, as humanity, have never been,” said Joe Westlake, the director of the Heliophysics Division at NASA Headquarters. “It’s an amazing accomplishment.”The spacecraft’s next planned close solar passes come on March 22, 2025, and June 19, 2025.By Mara Johnson-GrohNASA’s Goddard Space Flight Center, Greenbelt, Md.Media Contact: Sarah FrazierShareDetailsLast UpdatedDec 27, 2024EditorAbbey InterranteRelated TermsGoddard Space Flight CenterHeliophysicsHeliophysics DivisionParker Solar Probe (PSP)Science & ResearchScience Mission DirectorateSolar FlaresSolar WindSpace WeatherThe SunThe Sun & Solar PhysicsExplore More3 min readAstronomy Activation Ambassadors: A New EraArticle1 day ago 2 min readScience Done by Volunteers Highlighted at December’s American Geophysical Union MeetingArticle1 week ago 4 min read2024: NASA Armstrong Prepares for Future Innovative Research EffortsNASA’s Armstrong Flight Research Center in Edwards, California, is preparing today for tomorrow’s mission. Supersonic flight, next generation aircraft, advanced air mobility, climate changes, human exploration of space, and the next innovation are just some of the topics our researchers, engineers, and mission support teams focused on in 2024. NASA Armstrong began 2024 with the […]Article2 weeks ago Return to topNational Aeronautics and Space AdministrationNASA explores the unknown in air and space, innovates for the benefit of humanity, and inspires the world through discovery.About NASA's MissionJoin UsHomeNews & EventsMultimediaNASA+MissionsHumans in SpaceEarth & ClimateThe Solar SystemThe UniverseScienceAeronauticsTechnologyLearning ResourcesAbout NASANASA en EspañolFollow NASAMore NASA Social AccountsNASA NewslettersSitemapFor MediaPrivacy PolicyFOIANo FEAR ActOffice of the IGBudget & Annual ReportsAgency Financial ReportsContact NASAAccessibilityPage Last Updated:5 days agoPage Editor:SMD Content EditorsResponsible NASA Official for Science:Dana Bolles + diff --git a/compose.yaml b/compose.yaml index e2fcee6f..522b53cb 100644 --- a/compose.yaml +++ b/compose.yaml @@ -18,7 +18,7 @@ x-services-common: &services-common restart: no services: - # NEXT SERVICE PORT: 10008 + # NEXT SERVICE PORT: 10009 #----------------------------------------------------- # admin :10001 #----------------------------------------------------- @@ -59,6 +59,53 @@ services: DEBUG_LEVEL: debug GIN_MODE: debug + #----------------------------------------------------- + # e2e :10008 + #----------------------------------------------------- + e2e: + <<: *services-common + image: jemison/dev + # Simulate CF + # https://stackoverflow.com/questions/42345235/how-to-specify-memory-cpu-limit-in-docker-compose-version-3 + deploy: + resources: + limits: + memory: 64m + hostname: e2e.gov + # networks: + # some-network: + # aliases: + # - e2e.gov + build: + context: . + dockerfile: ./cmd/entree/Dockerfile + entrypoint: /home/vcap/app/cmd/e2e/run.sh + volumes: + - type: bind + source: . + target: /home/vcap/app + ports: + - 10008:8888 + # https://docs.docker.com/compose/how-tos/startup-order/ + depends_on: + nginx: + condition: service_started + jemison-queues-db: + condition: service_healthy + jemison-work-db: + condition: service_healthy + healthcheck: + test: curl --fail http://e2e:8888/heartbeat || exit 1 + interval: 60s + timeout: 180s + retries: 3 + start_period: 60s + environment: + ENV: "DOCKER" + PORT: 8888 + DEBUG_LEVEL: debug + GIN_MODE: debug + SCHEDULE: ${SCHEDULE:-""} #----------------------------------------------------- # entree :10007 diff --git a/config/domain64/README.md b/config/domain64/README.md new file mode 100644 index 00000000..23d408b5 --- /dev/null +++ b/config/domain64/README.md @@ -0,0 +1,10 @@ +# adding domains + +Until further notice... + +1. Add it to the appropriate `tld.libsonnet` file. E.g. add to `gov.libsonnet` if the domain is a .gov, etc. +2. Add it to the `schedules.libsonnet`. + +A CTRL-F to find the domain that proceeds the new domain in the schedules file makes adding it quick. + +We need this to be automated... \ No newline at end of file diff --git a/config/domain64/gov.libsonnet b/config/domain64/gov.libsonnet index 50d77057..59cd747d 100644 --- a/config/domain64/gov.libsonnet +++ b/config/domain64/gov.libsonnet @@ -2081,11 +2081,18 @@ local domains = { }, name: 'wwtg', }, + '0000DD': { + children: { + '000001': 'www', + }, + name: 'e2e', + }, }; assert util.getDomain('search', domains) == '0000B2'; assert util.getDomain('cloud', domains) == '000024'; assert util.getDomain('vote', domains) == '0000DA'; +assert util.getDomain('e2e', domains) == '0000DD'; { domains: domains, diff --git a/config/domain64/schedules.libsonnet b/config/domain64/schedules.libsonnet index e1ec1440..6631b3a7 100644 --- a/config/domain64/schedules.libsonnet +++ b/config/domain64/schedules.libsonnet @@ -990,6 +990,7 @@ "010000D900000100": "monthly", "010000DB00000100": "monthly", "010000DC00000100": "monthly", + "010000DD00000100": "monthly", "0200000100000100": "monthly", "0200000100000200": "monthly", "0200000100000300": "monthly", diff --git a/config/hosts/allowed_hosts.jsonnet b/config/hosts/allowed_hosts.jsonnet index 6ecfe374..084b0fbb 100644 --- a/config/hosts/allowed_hosts.jsonnet +++ b/config/hosts/allowed_hosts.jsonnet @@ -22,12 +22,23 @@ local lookupD64(fqdn) = d64[getTLD(fqdn)].FQDNToDomain64[fqdn]; local d64ToDec(d) = std.parseHex(d); +local fqdnToDec(fqdn) = d64ToDec(lookupD64(fqdn)); + assert lookupD64('cloud.gov') == '0100002400000000'; { + e2e: [ + [ + util.toDec('010000DD00000000'), + util.toDec('010000DDFFFFFF00'), + ], + ], + three: [[d64ToDec(lookupD64(fqdn)), d64ToDec(lookupD64(fqdn))] for fqdn in fqdn_include] + [[d64ToDec(p[0]), d64ToDec(p[1])] for p in ranges_to_include], + all: [[0, d64ToDec('FFFFFFFFFFFFFF00')]], + nih: [ [ util.toDec('0100008D00000000'), @@ -79,4 +90,5 @@ assert lookupD64('cloud.gov') == '0100002400000000'; + self.eighteeneff + self.digital + self.fedramp, + + self.e2e, } From 6531d25f65257d4f563bc14273c73a2e741ece72 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Wed, 1 Jan 2025 17:14:41 -0500 Subject: [PATCH 2/3] Serves static pages Now to integrate... --- cmd/e2e/main.go | 7 ++++-- config/services/container.jsonnet | 2 ++ config/services/e2e.libsonnet | 37 +++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 config/services/e2e.libsonnet diff --git a/cmd/e2e/main.go b/cmd/e2e/main.go index 8fcf206f..41d74f5d 100644 --- a/cmd/e2e/main.go +++ b/cmd/e2e/main.go @@ -7,6 +7,7 @@ import ( "github.com/GSA-TTS/jemison/internal/env" "github.com/GSA-TTS/jemison/internal/postgres" "github.com/GSA-TTS/jemison/internal/queueing" + "github.com/gin-gonic/gin" "go.uber.org/zap" ) @@ -17,6 +18,7 @@ var JDB *postgres.JemisonDB func main() { env.InitGlobalEnv(ThisServiceName) + s, _ := env.Env.GetUserService(ThisServiceName) engine := common.InitializeAPI() @@ -25,7 +27,8 @@ func main() { zap.L().Info("listening to the music of the spheres", zap.String("port", env.Env.Port)) - // Serve the static directory at /static - engine.Static("/static", "./static") + static_files_path := s.GetParamString("static_files_path") + engine.StaticFS("/static", gin.Dir(static_files_path, true)) + http.ListenAndServe(":"+env.Env.Port, engine) } diff --git a/config/services/container.jsonnet b/config/services/container.jsonnet index 57d0271c..71c0a92c 100644 --- a/config/services/container.jsonnet +++ b/config/services/container.jsonnet @@ -1,4 +1,5 @@ local A = import 'admin.libsonnet'; +local E2 = import 'e2e.libsonnet'; local EN = import 'entree.libsonnet'; local EX = import 'extract.libsonnet'; local F = import 'fetch.libsonnet'; @@ -20,6 +21,7 @@ local VCAP = import 'vcap_services.libsonnet'; EIGHT_SERVICES: { 'user-provided': [ A.container, + E2.container, EN.container, EX.container, F.container, diff --git a/config/services/e2e.libsonnet b/config/services/e2e.libsonnet new file mode 100644 index 00000000..fc986e75 --- /dev/null +++ b/config/services/e2e.libsonnet @@ -0,0 +1,37 @@ +local B = import 'base.libsonnet'; +local service = 'e2e'; + +local credentials = [ + [ + 'port', + { cf: 8080, container: 8888 }, + ], +]; + +local parameters = [ + [ + 'workers', + { cf: 10, container: 10 }, + ], + [ + 'debug_level', + { cf: 'warn', container: 'debug' }, + ], + [ + 'static_files_path', + { + cf: '/home/vcap/app/static', + container: '/home/vcap/app/cmd/e2e/static', + }, + ], +] + B.parameters; + +{ + creds:: [[service] + x for x in credentials], + params:: [[service] + x for x in parameters], + cf: B.params('credentials', 'cf', service, self.creds) + + B.params('parameters', 'cf', service, self.params), + container: { name: service } + + B.params('credentials', 'container', service, self.creds) + + B.params('parameters', 'container', service, self.params), +} From 1a75ebee3fd069e8d8d53ce8b2988658a5ad64f5 Mon Sep 17 00:00:00 2001 From: Matt Jadud Date: Thu, 2 Jan 2025 08:55:52 -0500 Subject: [PATCH 3/3] The service can now walk itself This lets fetch/walk/etc. hit the internal e2e container. --- cmd/e2e/main.go | 47 ++++++++++++++++++++++++++++++++++- cmd/e2e/templates/index.tmpl | 30 ++++++++++++++++++++++ compose.yaml | 13 +++++++--- config/services/e2e.libsonnet | 7 ++++++ internal/filtering/general.go | 11 ++++++++ 5 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 cmd/e2e/templates/index.tmpl diff --git a/cmd/e2e/main.go b/cmd/e2e/main.go index 41d74f5d..c619fc75 100644 --- a/cmd/e2e/main.go +++ b/cmd/e2e/main.go @@ -1,7 +1,9 @@ package main import ( + "log" "net/http" + "os" "github.com/GSA-TTS/jemison/internal/common" "github.com/GSA-TTS/jemison/internal/env" @@ -20,15 +22,58 @@ func main() { env.InitGlobalEnv(ThisServiceName) s, _ := env.Env.GetUserService(ThisServiceName) + // For the heartbeat engine := common.InitializeAPI() + template_files_path := s.GetParamString("template_files_path") + static_files_path := s.GetParamString("static_files_path") + + // For the index template + // This provides links to the HTML files + // in the static directory. + engine.LoadHTMLGlob(template_files_path + "/*") + + base_params := gin.H{ + "scheme": "http", + "search_host": "e2e.gov", + "search_port": env.Env.Port, + } + + files, err := os.ReadDir(static_files_path) + if err != nil { + log.Fatal(err) + } + + filenames := make([]string, 0) + for _, file := range files { + if !file.IsDir() { + filenames = append(filenames, file.Name()) + } + } + + engine.GET("/www", func(c *gin.Context) { + // d64_start, _ := strconv.ParseInt("010000DD00000000", 16, 64) + // d64_end, _ := strconv.ParseInt("010000DDFFFFFF00", 16, 64) + // base_params["d64_start"] = d64_start + // base_params["d64_end"] = d64_end + base_params["filenames"] = filenames + c.HTML(http.StatusOK, "index.tmpl", base_params) + }) + + engine.HEAD("/www", func(c *gin.Context) { + // c.Writer.Header().Set("Access-Control-Allow-Origin", "*") + base_params["filenames"] = filenames + + c.HTML(http.StatusOK, "index.tmpl", base_params) + }) + go queueing.Enqueue(ChQSHP) zap.L().Info("listening to the music of the spheres", zap.String("port", env.Env.Port)) - static_files_path := s.GetParamString("static_files_path") engine.StaticFS("/static", gin.Dir(static_files_path, true)) + go http.ListenAndServe(":80", engine) http.ListenAndServe(":"+env.Env.Port, engine) } diff --git a/cmd/e2e/templates/index.tmpl b/cmd/e2e/templates/index.tmpl new file mode 100644 index 00000000..1a6efd9c --- /dev/null +++ b/cmd/e2e/templates/index.tmpl @@ -0,0 +1,30 @@ + + + + + + + + + + +
+
+
+
+
+

E2E test pages

+

This is some content.

+
    + {{range .filenames}} +
  • {{ . }}
    • + {{end}} +
+
+
+
+
+ + + + \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 522b53cb..c82734e1 100644 --- a/compose.yaml +++ b/compose.yaml @@ -62,7 +62,7 @@ services: #----------------------------------------------------- # e2e :10008 #----------------------------------------------------- - e2e: + e2e.gov: <<: *services-common image: jemison/dev # Simulate CF @@ -71,7 +71,14 @@ services: resources: limits: memory: 64m - hostname: e2e.gov + # external_links: + # - test.gov:t2.gov + networks: + default: + aliases: + - www.e2e.gov + - end2end.gov + - endtoend.gov # networks: # some-network: # aliases: @@ -95,7 +102,7 @@ services: jemison-work-db: condition: service_healthy healthcheck: - test: curl --fail http://e2e:8888/heartbeat || exit 1 + test: curl --fail http://e2e.gov:8888/heartbeat || exit 1 interval: 60s timeout: 180s retries: 3 diff --git a/config/services/e2e.libsonnet b/config/services/e2e.libsonnet index fc986e75..2a1eda7a 100644 --- a/config/services/e2e.libsonnet +++ b/config/services/e2e.libsonnet @@ -13,6 +13,13 @@ local parameters = [ 'workers', { cf: 10, container: 10 }, ], + [ + 'template_files_path', + { + cf: '/home/vcap/app/templates', + container: '/home/vcap/app/cmd/e2e/templates', + }, + ], [ 'debug_level', { cf: 'warn', container: 'debug' }, diff --git a/internal/filtering/general.go b/internal/filtering/general.go index 078b28c5..8dee7c19 100644 --- a/internal/filtering/general.go +++ b/internal/filtering/general.go @@ -5,6 +5,8 @@ import ( "net/url" "regexp" "strings" + + "github.com/GSA-TTS/jemison/internal/env" ) var skippable_prefixes = []string{"#", "mailto"} @@ -52,6 +54,15 @@ func hasSlashHttp(u *url.URL) error { } func insecureGov(u *url.URL) error { + // If we are running against a fake domain, and in our local stack, + // we want this to pass. This will only work in the containerized env. + if env.IsContainerEnv() { + except, _ := regexp.MatchString(`^http://www.e2e.gov`, u.String()) + if except { + return nil + } + } + m, _ := regexp.MatchString(`^http:`, u.String()) if m { return fmt.Errorf("insecure URL: %s", u.String())