From 76006c4e151340be77b71d853471b84b109aa535 Mon Sep 17 00:00:00 2001 From: michaelkedar Date: Mon, 26 Oct 2026 04:36:47 +0000 Subject: [PATCH] chore: speed up Go tests and fix Datastore emulator leaks --- AGENTS.md | 1 - docker/ci/install_go.sh | 2 +- go/cloudbuild.yaml | 2 + go/cmd/relations/main_test.go | 25 +++ go/go.mod | 10 -- go/go.sum | 37 ---- go/internal/database/datastore/main_test.go | 25 +++ go/internal/recoverer/recoverer_test.go | 4 + go/internal/repos/repos.go | 165 ------------------ go/testutils/datastore.go | 184 ++++++++++++-------- go/testutils/vcr.go | 5 +- vulnfeeds/cloudbuild.yaml | 2 + vulnfeeds/internal/testutils/testutils.go | 5 +- 13 files changed, 176 insertions(+), 291 deletions(-) create mode 100644 go/cmd/relations/main_test.go create mode 100644 go/internal/database/datastore/main_test.go delete mode 100644 go/internal/repos/repos.go diff --git a/AGENTS.md b/AGENTS.md index c7f8a7275c9..5b2aa042859 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -304,7 +304,6 @@ All Go microservices are compiled using a single, unified multi-target Dockerfil - **`postgres/`**: PostgreSQL implementation of the domain interfaces using `sqlc` and `pgx/v5` ([`go/internal/database/postgres/`](go/internal/database/postgres/)). - **`dualread/`**: Shadow dual-reader wrapper ([`go/internal/database/dualread/`](go/internal/database/dualread/)) that serves primary reads from Datastore/GCS while probabilistically shadowing queries (`POSTGRES_SHADOW_RATE`) to PostgreSQL in the background for latency comparison. - **`gitter/`**: Client logic to interface with the Gitter caching service. -- **`repos/`**: Shared Git repository management and utilities. --- diff --git a/docker/ci/install_go.sh b/docker/ci/install_go.sh index 79cd95ab301..f590b379e01 100755 --- a/docker/ci/install_go.sh +++ b/docker/ci/install_go.sh @@ -19,4 +19,4 @@ set -eux # Download and install Go # https://go.dev/doc/install -curl https://go.dev/dl/go1.27.0.linux-amd64.tar.gz -LO && rm -rf /usr/local/go && tar -C /usr/local -xzf go1.27.0.linux-amd64.tar.gz +curl https://go.dev/dl/go1.27.1.linux-amd64.tar.gz -LO && rm -rf /usr/local/go && tar -C /usr/local -xzf go1.27.1.linux-amd64.tar.gz diff --git a/go/cloudbuild.yaml b/go/cloudbuild.yaml index 5bba80d9b51..c35584439e6 100644 --- a/go/cloudbuild.yaml +++ b/go/cloudbuild.yaml @@ -37,3 +37,5 @@ steps: waitFor: ['init'] timeout: 7200s +options: + machineType: E2_HIGHCPU_8 diff --git a/go/cmd/relations/main_test.go b/go/cmd/relations/main_test.go new file mode 100644 index 00000000000..c2c43033e74 --- /dev/null +++ b/go/cmd/relations/main_test.go @@ -0,0 +1,25 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/google/osv.dev/go/testutils" +) + +func TestMain(m *testing.M) { + testutils.RunWithDatastoreEmulator(m) +} diff --git a/go/go.mod b/go/go.mod index c623c270efe..09bd0b2fb1d 100644 --- a/go/go.mod +++ b/go/go.mod @@ -16,7 +16,6 @@ require ( github.com/alicebob/miniredis/v2 v2.40.0 github.com/dgraph-io/ristretto/v2 v2.4.2 github.com/dustin/go-humanize v1.1.0 - github.com/go-git/go-git/v6 v6.0.0-beta.1 github.com/google/go-cmp v0.7.0 github.com/google/osv-scalibr v0.5.3 github.com/hashicorp/go-retryablehttp v0.7.8 @@ -63,8 +62,6 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.60.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.60.0 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/ProtonMail/go-crypto v1.5.2 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -78,14 +75,10 @@ require ( github.com/charmbracelet/x/windows v0.2.2 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect - github.com/cloudflare/circl v1.6.3 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect - github.com/emirpasic/gods v1.18.1 // indirect github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.1.0 // indirect - github.com/go-git/gcfg/v2 v2.0.2 // indirect - github.com/go-git/go-billy/v6 v6.0.0-beta.1 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -100,14 +93,12 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kevinburke/ssh_config v1.6.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.1 // indirect github.com/mattn/go-runewidth v0.0.30 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pjbgf/sha1cd v0.7.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/prometheus/client_model v0.6.3 // indirect @@ -115,7 +106,6 @@ require ( github.com/prometheus/procfs v0.22.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sahilm/fuzzy v0.1.3 // indirect - github.com/sergi/go-diff v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/tidwall/match v1.2.0 // indirect diff --git a/go/go.sum b/go/go.sum index 5f65c17c8a5..1c6809962f5 100644 --- a/go/go.sum +++ b/go/go.sum @@ -40,16 +40,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.60.0/go.mod h1:SRAbhyZ4R4FagHMM9VtRgSY/lheRoht2fKelZXQUenk= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= -github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= -github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/ProtonMail/go-crypto v1.5.2 h1:cucYnvqcY7UOXVD//mSyjeaPY0SSN3v5cDkYPxumINk= -github.com/ProtonMail/go-crypto v1.5.2/go.mod h1:/RaSu30DaKO4RY+XdV/ACcCcZkGr7AhUIduq5sjzzCo= github.com/alicebob/miniredis/v2 v2.40.0 h1:6IgTQbvm2JiMYZ8yur/acNJzVjtSOWCelxXbk/LUBXc= github.com/alicebob/miniredis/v2 v2.40.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ12Gv5o= @@ -86,8 +78,6 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSE github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= -github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= -github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= @@ -99,8 +89,6 @@ github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da h1:aIftn67I1fkbMa5 github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= -github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= -github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -117,16 +105,6 @@ github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= -github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= -github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= -github.com/go-git/gcfg/v2 v2.0.2 h1:MY5SIIfTGGEMhdA7d7JePuVVxtKL7Hp+ApGDJAJ7dpo= -github.com/go-git/gcfg/v2 v2.0.2/go.mod h1:/lv2NsxvhepuMrldsFilrgct6pxzpGdSRC13ydTLSLs= -github.com/go-git/go-billy/v6 v6.0.0-beta.1 h1:OJ4r207zcEbeq/DJNajQUOMd+pph3r9A7hRq1Sjl1bA= -github.com/go-git/go-billy/v6 v6.0.0-beta.1/go.mod h1:QGOWN4CzmtRfpFeCw5paG3RJyKdTwsfGASkxBVkIaAU= -github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1 h1:gmqi2jvsreu0s8JMLylYDFq4sbjHwwlhktMw0DUg3mA= -github.com/go-git/go-git-fixtures/v6 v6.0.0-alpha.1/go.mod h1:ECf1MqJlBdYpKggBrOXjo/0EnvRZx6D++I86UYjPgAQ= -github.com/go-git/go-git/v6 v6.0.0-beta.1 h1:1t+vJYGDBN6f9GRWskgWx5UjXPFUa7wfffZZZ2C5GtA= -github.com/go-git/go-git/v6 v6.0.0-beta.1/go.mod h1:ugDlr38Wwc+fOw85J+Oluh5TNXQtJgYMcriOljg3l4s= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -198,17 +176,12 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= -github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4= github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -245,8 +218,6 @@ github.com/package-url/packageurl-go v0.1.7 h1:iFWg6tzAjLA6F/qX3M5nZaiMHJgc+p2zx github.com/package-url/packageurl-go v0.1.7/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= github.com/pandatix/go-cvss v0.6.4 h1:9w2RCO/Q4UTiJyEgpCHRiVc6CfrsFEnkoX+OtATqKio= github.com/pandatix/go-cvss v0.6.4/go.mod h1:/ukvQnYlrKl3o/DVp7/GO2UZyZheuo/maOK0U1nBEhQ= -github.com/pjbgf/sha1cd v0.7.0 h1:ZRNPKHj+gfkLBf0KJv/p1Hmz+7bqCT5o311YX0nq+DA= -github.com/pjbgf/sha1cd v0.7.0/go.mod h1:pKR5Li+qTCo+ebqITZfwPlJGoCFCvSO00qxjbNtTUFQ= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -269,8 +240,6 @@ github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/sahilm/fuzzy v0.1.3 h1:juByESSS32nVD81vr6tHmKmA/8zde7gE+x5CLxrzXPU= github.com/sahilm/fuzzy v0.1.3/go.mod h1:au6//VbVSqu6DFrkL2CfjlJ5iURpNCPeE+1GwY3XsT8= -github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= -github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spiffe/go-spiffe/v2 v2.8.1 h1:eXZMLsu+3MLEPJyGJkolqtVrteZfQdUpOWj6LTiDl/E= @@ -279,7 +248,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -382,8 +350,6 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= -golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= @@ -432,11 +398,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/dnaeon/go-vcr.v4 v4.0.7 h1:Mq/RF+mq3QwtEunJSsoTbYPt3elSAmdJhAxrEaqr88I= gopkg.in/dnaeon/go-vcr.v4 v4.0.7/go.mod h1:cRwV/njsN/D8qNJu4NAXWswz6b4OUh3rMIu4SObbLBg= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go/internal/database/datastore/main_test.go b/go/internal/database/datastore/main_test.go new file mode 100644 index 00000000000..68a9948ab49 --- /dev/null +++ b/go/internal/database/datastore/main_test.go @@ -0,0 +1,25 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "testing" + + "github.com/google/osv.dev/go/testutils" +) + +func TestMain(m *testing.M) { + testutils.RunWithDatastoreEmulator(m) +} diff --git a/go/internal/recoverer/recoverer_test.go b/go/internal/recoverer/recoverer_test.go index 0f282d9716f..0474e3a3a8f 100644 --- a/go/internal/recoverer/recoverer_test.go +++ b/go/internal/recoverer/recoverer_test.go @@ -38,6 +38,10 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +func TestMain(m *testing.M) { + testutils.RunWithDatastoreEmulator(m) +} + type mockGitterClient struct { gitter.Client diff --git a/go/internal/repos/repos.go b/go/internal/repos/repos.go deleted file mode 100644 index 17448117725..00000000000 --- a/go/internal/repos/repos.go +++ /dev/null @@ -1,165 +0,0 @@ -// Package repos provides utilities for managing and cloning source repositories. -package repos - -import ( - "context" - "errors" - "fmt" - "io" - "log/slog" - "net/http" - "net/url" - "os" - "os/exec" - "path/filepath" - "strings" - - "github.com/go-git/go-git/v6" - "github.com/google/osv.dev/go/logger" - "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" -) - -var ErrRepoInaccessible = errors.New("repo inaccessible") - -// More performant mirrors for large/popular repos. -var gitMirrors = map[string]string{ - "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git": "https://kernel.googlesource.com/pub/scm/linux/kernel/git/stable/linux.git", -} - -func gitMirror(gitURL string) string { - if mirror, ok := gitMirrors[strings.TrimRight(gitURL, "/")]; ok { - logger.Info("Using mirror for git URL", slog.String("mirror", mirror), slog.String("git_url", gitURL)) - return mirror - } - - return gitURL -} - -func CloneToDir(ctx context.Context, repoURL string, dir string, forceUpdate bool) (*git.Repository, error) { - // If the dir exists, check if it's a git repo. If so, pull the latest changes. - // If not, clone the repo into the dir. - _, err := os.Stat(dir) - if err == nil { - // Dir exists, check if it's a git repo. - repo, err := git.PlainOpen(dir) - if err == nil { - // It's a git repo, pull the latest changes. - if forceUpdate { - cmd := exec.CommandContext(ctx, "git", "fetch") - cmd.Dir = dir - cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0") - output, err := cmd.CombinedOutput() - if err != nil { - return nil, fmt.Errorf("failed to fetch repo: %w\n%s", err, string(output)) - } - logger.Info("Pulled latest changes", slog.String("repo_url", repoURL), slog.String("dir", dir), slog.String("output", string(output))) - - // re-open the repo to get the new HEAD - return git.PlainOpen(dir) - } - - return repo, nil - } - logger.Warn("Could not open existing directory as git repo. Deleting it and cloning from scratch", slog.String("dir", dir), slog.Any("error", err)) - if err := os.RemoveAll(dir); err != nil { - return nil, fmt.Errorf("failed to remove directory: %w", err) - } - } - if err := os.MkdirAll(filepath.Dir(dir), 0755); err != nil { - return nil, fmt.Errorf("failed to create directory: %w", err) - } - repoURL = gitMirror(repoURL) - if gitterHost := os.Getenv("GITTER_HOST"); gitterHost != "" { - repo, err := cloneToDirGitter(ctx, gitterHost, repoURL, dir, forceUpdate) - if err == nil { - return repo, nil - } - if errors.Is(err, ErrRepoInaccessible) { - return nil, err - } - logger.Error("Failed to get repo from gitter", slog.String("url", repoURL), slog.Any("error", err)) - logger.Info("Falling back to regular clone") - } - - // regular clone - cmd := exec.CommandContext(ctx, "git", "clone", "--", repoURL, dir) - cmd.Env = append(cmd.Env, "GIT_TERMINAL_PROMPT=0") - output, err := cmd.CombinedOutput() - if err != nil { - return nil, fmt.Errorf("failed to clone repo: %w\n%s", err, string(output)) - } - - return git.PlainOpen(dir) -} - -func cloneToDirGitter(ctx context.Context, gitterHost, repoURL, dir string, forceUpdate bool) (*git.Repository, error) { - w, err := gitterGet(ctx, gitterHost, repoURL, forceUpdate) - if err != nil { - return nil, fmt.Errorf("failed to get repo from gitter: %w", err) - } - // write tar.zst into file - defer w.Close() - tmpDir, err := os.MkdirTemp("", "gitter-*") - if err != nil { - return nil, fmt.Errorf("failed to create temp dir: %w", err) - } - tarPath := filepath.Join(tmpDir, "repo.tar.zst") - f, err := os.Create(tarPath) - if err != nil { - return nil, fmt.Errorf("failed to create tar file: %w", err) - } - defer f.Close() - _, err = io.Copy(f, w) - if err != nil { - return nil, fmt.Errorf("failed to write tar file: %w", err) - } - // shell out to tar xf - if err := os.MkdirAll(dir, 0755); err != nil { - return nil, fmt.Errorf("failed to create directory: %w", err) - } - cmd := exec.CommandContext(ctx, "tar", "-xf", tarPath, "-C", dir) - output, err := cmd.CombinedOutput() - if err != nil { - return nil, fmt.Errorf("failed to untar repo: %w\n%s", err, string(output)) - } - // clean up - if err := os.RemoveAll(tmpDir); err != nil { - logger.Error("Failed to remove temp dir", slog.String("dir", tmpDir), slog.Any("error", err)) - } - - return git.PlainOpen(dir) -} - -func gitterGet(ctx context.Context, gitterHost, repoURL string, forceUpdate bool) (io.ReadCloser, error) { - getGitURL, err := url.JoinPath(gitterHost, "git") - if err != nil { - return nil, fmt.Errorf("failed to join path: %w", err) - } - // Use a custom transport to add OpenTelemetry tracing. - client := http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport)} - //nolint:gosec // G704: Request to internal gitter service - req, err := http.NewRequestWithContext(ctx, http.MethodGet, getGitURL, nil) - if err != nil { - return nil, fmt.Errorf("failed to create request: %w", err) - } - vals := req.URL.Query() - vals.Set("url", repoURL) - if forceUpdate { - vals.Set("force-update", "true") - } - req.URL.RawQuery = vals.Encode() - logger.Info("Getting repo from gitter", slog.String("url", req.URL.String())) - //nolint:gosec // G704: Request to internal gitter service - resp, err := client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to get repo from gitter: %w", err) - } - if resp.StatusCode == http.StatusForbidden { - return nil, ErrRepoInaccessible - } - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to clone repo: %s", resp.Status) - } - - return resp.Body, nil -} diff --git a/go/testutils/datastore.go b/go/testutils/datastore.go index de89f155bb9..e119de47b36 100644 --- a/go/testutils/datastore.go +++ b/go/testutils/datastore.go @@ -25,6 +25,7 @@ import ( "os/exec" "strconv" "sync" + "syscall" "testing" "time" @@ -35,15 +36,33 @@ const ( datastoreEmulatorHost = "localhost" datastoreEmulatorProjectID = "test-project" startupTimeout = 30 * time.Second + httpTimeout = 10 * time.Second ) var ( - emulatorCmd *exec.Cmd - emulatorPort string - emulatorOnce sync.Once - emulatorErr error //nolint:errname // This is error state of the emulator, not an error type + emulatorPort string + emulatorHTTPClient = &http.Client{Timeout: httpTimeout} ) +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + + return b.buf.String() +} + func getFreePort() (string, error) { addr, err := net.ResolveTCPAddr("tcp", "localhost:0") if err != nil { @@ -61,11 +80,11 @@ func getFreePort() (string, error) { func resetEmulator(t *testing.T) { t.Helper() resetURL := fmt.Sprintf("http://%s/reset", net.JoinHostPort(datastoreEmulatorHost, emulatorPort)) - req, err := http.NewRequest(http.MethodPost, resetURL, nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, resetURL, nil) if err != nil { t.Fatalf("Failed to create reset request: %v", err) } - resp, err := http.DefaultClient.Do(req) + resp, err := emulatorHTTPClient.Do(req) if err != nil { t.Fatalf("Failed to send reset request: %v", err) } @@ -75,89 +94,104 @@ func resetEmulator(t *testing.T) { } } -// StartDatastoreEmulator ensures the Datastore emulator is running once per test execution. -func StartDatastoreEmulator(t *testing.T) { - t.Helper() - emulatorOnce.Do(func() { - port, err := getFreePort() - if err != nil { - emulatorErr = fmt.Errorf("failed to get free port: %w", err) - return - } - emulatorPort = port - - //nolint:gosec - emulatorCmd = exec.Command("gcloud", "emulators", "firestore", "start", - "--database-mode=datastore-mode", - fmt.Sprintf("--host-port=%s:%s", datastoreEmulatorHost, emulatorPort)) - - var output bytes.Buffer - emulatorCmd.Stdout = &output - emulatorCmd.Stderr = &output - - if err := emulatorCmd.Start(); err != nil { - emulatorErr = fmt.Errorf("failed to start datastore emulator: %w", err) - return - } +func startDatastoreEmulator() (*exec.Cmd, error) { + port, err := getFreePort() + if err != nil { + return nil, fmt.Errorf("failed to get free port: %w", err) + } + emulatorPort = port + hostPort := net.JoinHostPort(datastoreEmulatorHost, emulatorPort) + + //nolint:gosec + cmd := exec.Command("gcloud", "emulators", "firestore", "start", + "--database-mode=datastore-mode", + "--host-port="+hostPort) + // Start the emulator in its own process group so we can kill gcloud, + // its bash wrapper, and the underlying Java process together. + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + var output syncBuffer + cmd.Stdout = &output + cmd.Stderr = &output + + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start datastore emulator: %w", err) + } - // Wait for the emulator to be ready - ctx, cancel := context.WithTimeout(context.Background(), startupTimeout) - defer cancel() - - ready := make(chan struct{}) - go func() { - for { - select { - case <-ctx.Done(): - return - default: - } - conn, err := net.DialTimeout("tcp", net.JoinHostPort(datastoreEmulatorHost, emulatorPort), 100*time.Millisecond) + ctx, cancel := context.WithTimeout(context.Background(), startupTimeout) + defer cancel() + + healthURL := "http://" + hostPort + "/" + ready := make(chan struct{}) + go func() { + for { + select { + case <-ctx.Done(): + return + default: + } + reqCtx, reqCancel := context.WithTimeout(ctx, 500*time.Millisecond) + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, healthURL, nil) + if err == nil { + resp, err := emulatorHTTPClient.Do(req) if err == nil { - conn.Close() - close(ready) + _ = resp.Body.Close() + if resp.StatusCode == http.StatusOK { + reqCancel() + close(ready) - return + return + } } - time.Sleep(100 * time.Millisecond) } - }() - - select { - case <-ctx.Done(): - emulatorErr = fmt.Errorf("datastore emulator did not start in time: %w\nOutput:\n%s", ctx.Err(), output.String()) - return - case <-ready: - t.Log("Datastore emulator is ready") + reqCancel() + time.Sleep(100 * time.Millisecond) } + }() - t.Cleanup(func() { - t.Log("Stopping Datastore emulator") - if emulatorCmd != nil && emulatorCmd.Process != nil { - if err := emulatorCmd.Process.Kill(); err != nil { - t.Logf("Failed to kill datastore emulator: %v", err) - } - _ = emulatorCmd.Wait() - } - os.Unsetenv("DATASTORE_EMULATOR_HOST") - os.Unsetenv("DATASTORE_PROJECT_ID") - }) - }) + select { + case <-ctx.Done(): + stopDatastoreEmulator(cmd) - if emulatorErr != nil { - t.Fatalf("Datastore emulator setup failed: %v", emulatorErr) + return nil, fmt.Errorf("datastore emulator did not start in time: %w\nOutput:\n%s", ctx.Err(), output.String()) + case <-ready: } - t.Setenv("DATASTORE_EMULATOR_HOST", net.JoinHostPort(datastoreEmulatorHost, emulatorPort)) - t.Setenv("DATASTORE_PROJECT_ID", datastoreEmulatorProjectID) - resetEmulator(t) + os.Setenv("DATASTORE_EMULATOR_HOST", hostPort) + os.Setenv("DATASTORE_PROJECT_ID", datastoreEmulatorProjectID) + + return cmd, nil +} + +func stopDatastoreEmulator(cmd *exec.Cmd) { + if cmd != nil && cmd.Process != nil { + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + _ = cmd.Wait() + } +} + +// RunWithDatastoreEmulator starts the Datastore emulator for a test package, +// runs m.Run(), stops the emulator process group, and exits with the test status code. +func RunWithDatastoreEmulator(m *testing.M) { + cmd, err := startDatastoreEmulator() + if err != nil { + fmt.Fprintf(os.Stderr, "Datastore emulator setup failed: %v\n", err) + os.Exit(1) + } + code := m.Run() + stopDatastoreEmulator(cmd) + os.Exit(code) } -// MustNewDatastoreClientForTesting creates a new Datastore client connected to the emulator. -// It ensures the emulator is started and cleaned up. +// MustNewDatastoreClientForTesting creates a new Datastore client connected to the emulator +// and resets the emulator state for the test. +// The package's TestMain must call RunWithDatastoreEmulator. func MustNewDatastoreClientForTesting(t *testing.T) *datastore.Client { t.Helper() - StartDatastoreEmulator(t) + if emulatorPort == "" { + t.Fatal("Datastore emulator is not running; call testutils.RunWithDatastoreEmulator(m) in TestMain") + } + resetEmulator(t) client, err := datastore.NewClient(context.Background(), datastoreEmulatorProjectID) if err != nil { diff --git a/go/testutils/vcr.go b/go/testutils/vcr.go index 8ab0f0f98cc..0977f9ea682 100644 --- a/go/testutils/vcr.go +++ b/go/testutils/vcr.go @@ -20,7 +20,10 @@ func SetupVCR(t *testing.T) *recorder.Recorder { for _, c := range illegalChars { cassetteName = strings.ReplaceAll(cassetteName, string(c), "_") } - r, err := recorder.New(filepath.Join("testdata", cassetteName)) + r, err := recorder.New( + filepath.Join("testdata", cassetteName), + recorder.WithSkipRequestLatency(true), + ) if err != nil { t.Fatal(err) } diff --git a/vulnfeeds/cloudbuild.yaml b/vulnfeeds/cloudbuild.yaml index 9c72aabd3df..31decf56750 100644 --- a/vulnfeeds/cloudbuild.yaml +++ b/vulnfeeds/cloudbuild.yaml @@ -37,3 +37,5 @@ steps: waitFor: ['init'] timeout: 7200s +options: + machineType: E2_HIGHCPU_8 diff --git a/vulnfeeds/internal/testutils/testutils.go b/vulnfeeds/internal/testutils/testutils.go index 27df2f003a4..f9ea5526f60 100644 --- a/vulnfeeds/internal/testutils/testutils.go +++ b/vulnfeeds/internal/testutils/testutils.go @@ -23,7 +23,10 @@ func SetupVCR(t *testing.T) *recorder.Recorder { for _, c := range illegalChars { cassetteName = strings.ReplaceAll(cassetteName, string(c), "_") } - r, err := recorder.New(filepath.Join("testdata", cassetteName)) + r, err := recorder.New( + filepath.Join("testdata", cassetteName), + recorder.WithSkipRequestLatency(true), + ) if err != nil { t.Fatal(err) }