DEV-1291 (b): Adds example for using rewriteMap to restrict maps acce…

compose.yml

@@ -3,8 +3,11 @@
 services:
   app:
     build: .
+    environment:
+      - MAPS_ROOT=/var/www/html/maps
     ports:
       - 8080:8080
     volumes:
       - ./test/conf/cas-vhost.conf:/etc/httpd/conf.d/cas-vhost.conf:ro
+      - ./test/conf/map-access.py:/usr/local/bin/map-access.py
       - ./test/data:/var/www/html:ro

test/conf/cas-vhost.conf

@@ -1,3 +1,24 @@
+Mutex file:/var/run/httpd rewrite-map
+
+RewriteEngine On
+RewriteMap mapaccess "prg:/usr/local/bin/map-access.py ${MAPS_ROOT}" www-data:www-data
+RewriteCond "${mapaccess:$1}" "=Public"
+RewriteRule "^/maps/([^/]+)/data/" - [E=MAP_IS_PUBLIC:1]
+
+# Only data files are restricted. Directory listings (and metadata) are always
+# public so users can browse a map even when they can't download it.
+<LocationMatch "^/maps/[^/]+/data/">
+    # mod_dir's index.html subrequest would otherwise trigger the CAS redirect.
+    DirectoryIndex disabled
+    AuthType CAS
+    CASScope /maps
+    <RequireAny>
+        Require expr -d %{REQUEST_FILENAME}
+        Require env MAP_IS_PUBLIC
+        Require valid-user
+    </RequireAny>
+</LocationMatch>
+
 <Location /UCB>
   AuthType CAS
   Require valid-user

test/conf/map-access.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+
+"""
+Returns
+"""
+
+import json
+import os
+import re
+import sys
+
+from collections import namedtuple, OrderedDict
+from pathlib import Path
+
+
+# mod_rewrite starts prg: maps with an empty environment, so httpd passes the
+# root as an argument (expanded from ${MAPS_ROOT} in the config).
+MAPS_ROOT = Path(sys.argv[1] if len(sys.argv) > 1 else os.environ.get("MAPS_ROOT", "/srv/geospatial")).resolve()
+PUBLIC = b"Public\n"
+RESTRICTED = b"Restricted\n"
+VALID_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,254}$")
+CACHE_MAX = 45_000
+
+
+class RightsCache:
+    def __init__(self, max_items=CACHE_MAX):
+        self._cache = OrderedDict()
+        self._max_items = max_items
+
+    def set(self, map_id, rights, meta_stat):
+        key = self._cachekey(meta_stat)
+        self._cache[map_id] = (key, rights)
+        self._cache.move_to_end(map_id)
+        self._prune()
+
+    def get(self, map_id, meta_stat):
+        cached = self._cache.get(map_id)
+        if cached and cached[0] == self._cachekey(meta_stat):
+            self._cache.move_to_end(map_id)
+            return cached[1]
+        else:
+            return None
+
+    def rm(self, map_id):
+        self._cache.pop(map_id, None)
+
+    def _cachekey(self, meta_stat):
+        return (
+            meta_stat.st_dev,
+            meta_stat.st_ino,
+            meta_stat.st_size,
+            meta_stat.st_mtime_ns,
+            meta_stat.st_ctime_ns
+        )
+
+    def _prune(self):
+        while len(self._cache) > self._max_items:
+            self._cache.popitem(last=False)
+
+
+def log(msg):
+    try:
+        sys.stderr.write(f"map-access: {msg}\n")
+        sys.stderr.flush()
+    except Exception:
+        pass
+
+
+def get_access_rights(map_id: str, rights_cache: RightsCache) -> bytes:
+    log(f"Checking map: {map_id}")
+
+    if not VALID_ID.match(map_id):
+        return RESTRICTED
+
+    geoblacklight_json = Path(MAPS_ROOT / map_id / "metadata" / "geoblacklight.json").resolve()
+    log(f"Checking geoblacklight.json: {geoblacklight_json}")
+
+    # Guard against symlinks pointing outside the maps tree.
+    if not geoblacklight_json.is_relative_to(MAPS_ROOT):
+        log(f"[restricted] path escapes root: {map_id} -> {geoblacklight_json}")
+        return RESTRICTED
+
+    # Ensure metadata exists
+    try:
+        st = geoblacklight_json.stat()
+    except OSError as e:
+        log(f"[restricted] error stat-ing {geoblacklight_json}: {e!r}")
+        rights_cache.rm(map_id)
+        return RESTRICTED
+
+    if access_rights := rights_cache.get(map_id, st):
+        return access_rights
+
+    # Read accessRights from geoblacklight.json
+    try:
+        with geoblacklight_json.open('rb') as fh:
+            doc = json.load(fh)
+
+        access_rights = PUBLIC if is_public(doc) else RESTRICTED
+        log(f"Determined access rights from file: {map_id} -> {access_rights}")
+    except Exception as e:
+        log(f"could not read {geoblacklight_json}: {e!r}")
+        access_rights = RESTRICTED
+
+    rights_cache.set(map_id, access_rights, st)
+
+    return access_rights
+
+
+def is_public(doc):
+    value = doc.get('dct_accessRights_s', doc.get('dc_rights_s'))
+    return isinstance(value, str) and value.strip() == "Public"
+
+
+def main():
+    stdin = sys.stdin.buffer
+    stdout = sys.stdout.buffer
+    rights_cache = RightsCache()
+
+    while True:
+        line = stdin.readline()
+        if not line:
+            return
+
+        try:
+            map_id = line.decode('utf-8', 'strict').strip()
+            access_rights = get_access_rights(map_id, rights_cache=rights_cache)
+        except Exception as e:
+            log(f"Unexpected error: {e!r}")
+            access_rights = RESTRICTED
+
+        try:
+            stdout.write(access_rights)
+            stdout.flush()
+        except BrokenPipeError:
+            return
+
+if __name__ == "__main__":
+    main()

test/data/maps/berkeley-s76m41/metadata/geoblacklight.json

@@ -0,0 +1,30 @@
+{
+    "dc_creator_sm": [
+        "Geographic Information Systems, Sonoma County (Calif.)., California "
+    ],
+    "dc_description_s": "The local area development guideline dataset represents the discreet geographical areas subject to specific development guidelines within Sonoma County. The local area development guidelines and goals are as follows: Canon Manor West Subdivision Development Guideline The boundary polygon represents the Canon Manor West Subdivision Development Guideline, 237+/- acres located southwest of the intersection of Petaluma Hill Road and East Cotati Avenue, adopted by Ordinance #5675 & #5676 on July 18, 2006. The Canon Manor West Subdivision Development Guideline boundary surrounds those parcels subject to water conservation measures as described in Ordinance #5676. To note, Ordinance #5675 outlines the approved rezoning of the subject area [refer to adopted documentation for further insight]. - - - - Glen Ellen Development and Design Guideline, Subarea I & II The boundary polygon represents the Glen Ellen Development and Design Guideline adopted by Ordinance #4297 on November 14, 1990. The Glen Ellen Development and Design Guidelines boundary surrounds those parcels within subareas I and II of the Glen Ellen Study Area. The following applications within the subareas shall be evaluated in accordance with the Glen Ellen Development and Design Guidelines: Subarea I (exterior building permits, discretionary permits, and projects subject to administrative design review), and Subarea II (discretionary permits). The goal of the Glen Ellen Development and Design Guidelines is to direct development so that the character of Glen Ellen is enhanced and maintained. The guideline defines a local area, as a subunit of a planning area, which will provide a greater level of detail for specific uses or development in a local area. The Glen Ellen Development and Design Guidelines represent revised goals, objectives, policies and design guideline contained in previous versions of the Glen Ellen Town Plan (February 1988 and November 1988). - - - - Highway 12 Development Guideline The boundary polygon represents the Highway 12 Development Guideline adopted by Ordinance #5120 on August 25, 1998. The Highway 12 Development Guideline is applicable to parcels having frontage on Highway 12 from its intersection at El Verano Avenue, north, to its intersection at Agua Caliente Road. The Highway 12 Development Guideline is in conjunction with the Sonoma Valley Redevelopment Project Area. The goal of the Highway 12 Development Guideline is to provide criteria for private development to ensure the consistency of each individual project with the overall character of the Hwy 12 corridor. - - - - Highway 116 Scenic Corridor Study Guideline The boundary polygon represents the 116 Scenic Highway Corridor Study Guideline adopted by the State of California's Department of Transportation (Caltrans) on September 20, 1988. The goal of the 116 Scenic Highway Corridor Study is to protect and enhance the corridor. Therefore, the corridor limits are principally determined by topography (land forms) and/or large, dense stands of vegetation limiting views from the highway. If vegetation or man-made structures do not obstruct views from the highway, the farthest ridge lines are the limits of the corridor. Only those ridges visible from the corridor are included; consequently, areas behind ridges are not included. The width of the corridor varies greatly from as little as 100 feet to as much as 14,000 feet. Property within the corridor could be subject to higher levels of planning controls to preserve the scenic quality. - - - - Penngrove Main Street Design Guidelines The boundary polygon represents the Penngrove Main Street Design Guidelines adopted by Ordinance #5920 on December 14, 2010. The goal of the Penngrove Main Street Design Guidelines is to evaluate the merits of projects located in and around Main Street in Penngrove, between Old Redwood Highway and Adobe Road, and as specifically depicted by Figure E of Ordinance # 5920 article. Building permits involved new construction, or the exterior modification of existing structures, are subject to review to ensure conformance with the Penngrove Main Street Design Guidelines. In addition, all projects requiring discretionary entitlements, such as Design Review approval, Zoning Permits, and Use Permits, must comply with the Penngrove Main Street Design Guidelines. - - - - Russian River Corridor Design Guidelines The boundary polygon represents the Russian River Corridor Design Guidelines adopted by Ordinance #5921 on December 14, 2010. The goal of the Russian River Corridor Design Guidelines is to evaluate the merits of projects located within the central corridor which runs through the Russian River Redevelopment Project Area, and as specifically depicted in Figure A of Ordinance #5921 article. The Russian River Corridor Design Guidelines apply to commercial and multi-family residential projects. Single family projects are only subject to the Design Guidelines when involving four or more units. Building permits involving new construction, or any exterior modification of existing structures, are subject to review to ensure conformance with the Russian River Corridor Design Guidelines, as are all projects which require discretionary entitlements such as Design Review, Zoning Permits, and Use Permits. - - - - Sebastopol Road Urban Vision Plan The boundary polygon represents the Sebastopol Road Urban Vision Plan adopted by Resolution #07-0481 on October 31, 2007. The Urban Vision Plan for Sebastopol Road Corridor affects that stretch of Sebastopol Road between Dutton Avenue to the east and Stony Point Road to the west, linking both sides of Sebastopol Road as well as the area north of Sebastopol Road, ending at the Highway 12 right-of-way. The corridor is within the community of Roseland in the southwest area of Santa Rosa, California. While portions of the subject area are within the boundaries of the City of Santa Rosa, the majority of the properties are located in the unincorporated area of Sonoma County. The goal of the Sebastopol Road Urban Vision Plan is to create a consensus among the local community as to what future development of the corridor should look like and what types of uses should be located there. The goal includes land use concept and urban design criteria for the greater planning area, street scape enhancements, and envisioning the Roseland Village Shopping Center and the Dutton Plaza as an ideal location for a neighborhood center. - - - - Taylor/Sonoma/Mayacamas Design Guidelines The boundary polygon represents the Taylor/Sonoma/Mayacamas Design Guidelines adopted by Ordinance #5935 on May 10, 2011. The adoption amends Section 26-90-050 of the Sonoma County code to replace the existing Taylor-Sonoma Mountain Development Guidelines due to following General Plan scenic landscape units. The boundary polygon was revised with Resolution #11-0462 adopted August 23, 2011. The goal of the Taylor-Sonoma Mountain Development Guideline is to reduce the visual impacts of residential related development within the Scenic Landscape Units of Taylor, Sonoma, and Mayacamas areas as visible from public roads.",
+    "dc_format_s": "Shapefile",
+    "dc_identifier_s": "http://spatial.lib.berkeley.edu/viewpublic/berkeley-s76m41",
+    "dc_rights_s": "Public",
+    "dc_subject_sm": [
+        "Boundaries",
+        "Planningcadastre",
+        "Development",
+        "Design"
+    ],
+    "dc_title_s": "Local Area Development, Sonoma County, California, 2012",
+    "dc_type_s": "Dataset",
+    "dct_issued_dt": "2012-12-01T23:34:35.206Z",
+    "dct_provenance_s": "Berkeley",
+    "dct_references_s": "{\"http://www.opengis.net/def/serviceType/ogc/wfs\":\"https://geoservices.lib.berkeley.edu/geoserver/wfs\",\"http://www.opengis.net/def/serviceType/ogc/wms\":\"https://geoservices.lib.berkeley.edu/geoserver/wms\",\"http://www.isotc211.org/schemas/2005/gmd/\":\"https://spatial.lib.berkeley.edu/public/berkeley-s76m41/iso19139.xml\",\"http://schema.org/downloadUrl\":\"https://spatial.lib.berkeley.edu/public/berkeley-s76m41/data.zip\"}",
+    "dct_spatial_sm": [
+        "Sonoma County (Calif.)"
+    ],
+    "geoblacklight_version": "1.0",
+    "layer_geom_type_s": "Polygon",
+    "layer_id_s": "s76m41",
+    "layer_modified_dt": "2018-04-02T23:34:35.206Z",
+    "layer_slug_s": "berkeley-s76m41",
+    "solr_geom": "ENVELOPE(-123.110182, -122.363334, 38.530869, 38.21665)",
+    "solr_year_i": 2012
+}

test/data/maps/berkeley-s7c980/metadata/geoblacklight.json

@@ -0,0 +1,32 @@
+{
+  "dct_creator_sm":["All China Marketing Research Co., Ltd."],
+  "dct_description_sm":["This polygon dataset represents county boundaries in China from 1989. The attribute table includes the GB codes and names in Chinese and English (Pinyin). All data is copyrighted by China Data Center and/or its suppliers. Derived works that include the source data must be merged with other value-added data in such a way that the derived work cannot be converted back to the original source data format. This data is licensed by UC Berkeley for research, educational, and other non-commercial use by authorized users, which include persons affiliated with UC Berkeley and walk-in users who must access the data in person at the library.The data sources for the layers in the collection \"Administrative Boundary Maps of China, 1949-2014\" include the following primary sources: “The Historical Changes of Administrative Regions of China, 1949-1983” (three volumes); 《中华人民共和国县级以上行政区沿革》（三册，1949-1983年）“The Atlas of Changes of Administrative Regions of China”;《中华人民共和国行政区划沿革地图集》The yearly publications of “The Administrative Regions”, 1985-2014; 每年的出版行政区划简册（1985-2014年）The web site for administrative regions: www.xzqh.org; 行政区划网www.xzqh.org"],
+  "dct_format_s":"Shapefile",
+  "dct_identifier_sm":["https://spatial.lib.berkeley.edu/viewpublic/berkeley-s7c980"],
+  "dct_publisher_sm":["China Data Center, University of Michigan"],
+  "dct_accessRights_s":"Restricted",
+  "dct_subject_sm":["Boundaries","Administrative and Political Divisions"],
+  "dct_title_s":"County Boundaries, China, 1989",
+  "dct_isPartOf_sm":["berkeley-s7r712"],
+  "dct_issued_s":"2016-01-01",
+  "schema_provider_s":"University of California Berkeley",
+  "dct_references_s":"{\"http://www.opengis.net/def/serviceType/ogc/wfs\":\"https://geoservices-secure.lib.berkeley.edu/geoserver/wfs\",\"http://www.opengis.net/def/serviceType/ogc/wms\":\"https://geoservices-secure.lib.berkeley.edu/geoserver/wms\",\"http://www.isotc211.org/schemas/2005/gmd/\":\"https://spatial.lib.berkeley.edu/metadata/berkeley-s7c980/iso19139.xml\",\"http://schema.org/downloadUrl\":\"https://spatial.lib.berkeley.edu/UCB/berkeley-s7c980/data.zip\"}",
+  "dct_spatial_sm":["China"],
+  "gbl_resourceType_sm":["Polygon data","Vector data"],
+  "gbl_wxsIdentifier_s":"s7c980",
+  "gbl_mdModified_dt":"2019-06-26T23:34:35.206Z",
+  "id":"berkeley-s7c980",
+  "locn_geometry":"ENVELOPE(68.196555, 136.429552, 54.139796, 5.036617)",
+  "dcat_bbox":"ENVELOPE(68.196555, 136.429552, 54.139796, 5.036617)",
+  "solr_bboxtype":"ENVELOPE(68.196555, 136.429552, 54.139796, 5.036617)",
+  "gbl_indexYear_im":[1989],
+  "gbl_mdVersion_s":"Aardvark",
+  "gbl_resourceClass_sm":["Datasets"],
+  "_version_":1870817623581655040,
+  "timestamp":"2026-07-15T21:25:18.023Z",
+  "solr_bboxtype__minX":68.196555,
+  "solr_bboxtype__minY":5.036617,
+  "solr_bboxtype__maxX":136.429552,
+  "solr_bboxtype__maxY":54.139796,
+  "score":4.6867046
+}

test/run_tests.sh

@@ -1,37 +1,75 @@
 #!/bin/bash
 
 app_url="localhost:8080"
+cas_login="Location: https://auth.berkeley.edu/cas/"
 
-function test_public {
-  if curl -s "${app_url}/public/" | grep -q 'The Regents of the University of California'; then
-    echo "PASS: /public/ serves custom dynamic index page"
+# Restricted per metadata (dct_accessRights_s) and public (legacy dc_rights_s)
+restricted_map="/maps/berkeley-s7c980"
+public_map="/maps/berkeley-s76m41"
+
+function check {
+  local desc="$1"; shift
+  if "$@"; then
+    echo "PASS: ${desc}"
   else
-    echo "FAILED: /public/ serves custom dynamic index page"
+    echo "FAILED: ${desc}"
     return 1
   fi
 }
 
+function status_is {
+  [ "$(curl -s -o /dev/null -w '%{http_code}' "${app_url}$1")" = "$2" ]
+}
+
+function redirects_to_cas {
+  curl -sI "${app_url}$1" | grep -q "${cas_login}"
+}
+
+function lists {
+  curl -s "${app_url}$1" | grep -q "href=\"$2\""
+}
+
+function test_public {
+  check "/public/ serves custom dynamic index page" \
+    sh -c "curl -s '${app_url}/public/' | grep -q 'The Regents of the University of California'"
+}
+
 function test_ucb {
-  if curl -sI "${app_url}/UCB/" | grep -q "Location: https://auth.berkeley.edu/cas/"; then
-    echo "PASS: /UCB/ redirects to auth.berkeley.edu"
+  check "/UCB/ redirects to auth.berkeley.edu" redirects_to_cas /UCB/
+}
+
+function test_maps {
+  local rc=0
+  check "/maps/ lists maps"                      lists /maps/ berkeley-s7c980/                     || rc=1
+  check "restricted map: metadata is public"     status_is "${restricted_map}/metadata/geoblacklight.json" 200 || rc=1
+  check "restricted map: data/ listing is public" lists "${restricted_map}/data/" original/          || rc=1
+  check "restricted map: nested listing shows files" lists "${restricted_map}/data/original/" data.zip || rc=1
+  check "restricted map: data file requires CAS" redirects_to_cas "${restricted_map}/data/original/data.zip" || rc=1
+  check "restricted map: MultiViews can't bypass CAS" redirects_to_cas "${restricted_map}/data/original/data" || rc=1
+  check "public map: data file is served"        status_is "${public_map}/data/original/data.zip" 200 || rc=1
+  check "public map: repeat request (cached) is served" status_is "${public_map}/data/original/data.zip" 200 || rc=1
+  check "unknown map: data path requires CAS"    redirects_to_cas "/maps/nonexistent/data/x.zip"   || rc=1
+  return $rc
+}
+
+function test_map_access_unit {
+  # Run as www-data (like httpd does) so permission tests aren't skipped for root.
+  if docker compose run --rm --no-deps -T -u www-data -v "$PWD/test:/test:ro" app \
+      python3 -m unittest discover -s /test/unit; then
+    echo "PASS: map-access.py unit tests"
   else
-    echo "FAILED: /UCB/ redirects to auth.berkeley.edu"
+    echo "FAILED: map-access.py unit tests"
     return 1
   fi
 }
 
 function run_tests {
-  test_public &
-  test1=$!
-  test_ucb &
-  test2=$!
-
-  wait $test1
-  res1=$?
-  wait $test2
-  res2=$?
-
-  [ "$res1" -eq "0" -a "$res2" -eq "0" ] || exit 1
+  local rc=0
+  test_public || rc=1
+  test_ucb || rc=1
+  test_maps || rc=1
+  test_map_access_unit || rc=1
+  return $rc
 }
 
 function stackup {
@@ -46,7 +84,7 @@ function stackdown {
 function main {
   trap stackdown EXIT
   stackup
-  run_tests
+  run_tests || exit 1
 }
 
 main