From 69ffce8a79947bf9a2cccd20f082831c12f9961a Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Sat, 14 May 2022 13:46:23 -0400 Subject: [PATCH] Authenticate requests to remote S3 endpoint --- docker-compose.yml | 9 ++ docker/web/Dockerfile | 18 ++-- docker/web/aws-signature.lua | 149 +++++++++++++++++++++++++++++++ docker/web/nginx.conf | 167 ++++++++++++++++++++++++----------- 4 files changed, 285 insertions(+), 58 deletions(-) create mode 100644 docker/web/aws-signature.lua diff --git a/docker-compose.yml b/docker-compose.yml index dd5fe631..9c87c458 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -87,8 +87,17 @@ services: build: context: . dockerfile: ./docker/web/Dockerfile + args: + - APP_DIR=/srv/philomena + - S3_SCHEME=http + - S3_HOST=files + - S3_PORT=80 + - S3_BUCKET=philomena volumes: - .:/srv/philomena + environment: + - AWS_ACCESS_KEY_ID=local-identity + - AWS_SECRET_ACCESS_KEY=local-credential logging: driver: "none" depends_on: diff --git a/docker/web/Dockerfile b/docker/web/Dockerfile index e76dab63..cf267ee0 100644 --- a/docker/web/Dockerfile +++ b/docker/web/Dockerfile @@ -1,7 +1,15 @@ -FROM nginx:1.21.4-alpine -ENV APP_DIR /srv/philomena -ENV BUCKET_NAME philomena +FROM openresty/openresty:1.19.9.1-12-alpine +ARG APP_DIR +ARG S3_SCHEME +ARG S3_HOST +ARG S3_PORT +ARG S3_BUCKET + +RUN apk add --no-cache gettext curl perl && opm get jkeys089/lua-resty-hmac=0.06 && mkdir -p /etc/nginx/lua +COPY docker/web/aws-signature.lua /etc/nginx/lua COPY docker/web/nginx.conf /tmp/docker.nginx -RUN envsubst '$APP_DIR $BUCKET_NAME' < /tmp/docker.nginx > /etc/nginx/conf.d/default.conf +RUN envsubst '$APP_DIR $S3_SCHEME $S3_HOST $S3_PORT $S3_BUCKET' < /tmp/docker.nginx > /etc/nginx/conf.d/default.conf && \ + echo 'env AWS_ACCESS_KEY_ID;' >> /usr/local/openresty/nginx/conf/nginx.conf && \ + echo 'env AWS_SECRET_ACCESS_KEY;' >> /usr/local/openresty/nginx/conf/nginx.conf EXPOSE 80 -CMD ["nginx", "-g", "daemon off;"] +CMD ["openresty", "-g", "daemon off;"] diff --git a/docker/web/aws-signature.lua b/docker/web/aws-signature.lua new file mode 100644 index 00000000..fae28992 --- /dev/null +++ b/docker/web/aws-signature.lua @@ -0,0 +1,149 @@ +--[[ +Copyright 2018 JobTeaser + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--]] + +local cjson = require('cjson') +local resty_hmac = require('resty.hmac') +local resty_sha256 = require('resty.sha256') +local str = require('resty.string') + +local _M = { _VERSION = '0.1.2' } + +local function get_credentials () + local access_key = os.getenv('AWS_ACCESS_KEY_ID') + local secret_key = os.getenv('AWS_SECRET_ACCESS_KEY') + + return { + access_key = access_key, + secret_key = secret_key + } +end + +local function get_iso8601_basic(timestamp) + return os.date('!%Y%m%dT%H%M%SZ', timestamp) +end + +local function get_iso8601_basic_short(timestamp) + return os.date('!%Y%m%d', timestamp) +end + +local function get_derived_signing_key(keys, timestamp, region, service) + local h_date = resty_hmac:new('AWS4' .. keys['secret_key'], resty_hmac.ALGOS.SHA256) + h_date:update(get_iso8601_basic_short(timestamp)) + k_date = h_date:final() + + local h_region = resty_hmac:new(k_date, resty_hmac.ALGOS.SHA256) + h_region:update(region) + k_region = h_region:final() + + local h_service = resty_hmac:new(k_region, resty_hmac.ALGOS.SHA256) + h_service:update(service) + k_service = h_service:final() + + local h = resty_hmac:new(k_service, resty_hmac.ALGOS.SHA256) + h:update('aws4_request') + return h:final() +end + +local function get_cred_scope(timestamp, region, service) + return get_iso8601_basic_short(timestamp) + .. '/' .. region + .. '/' .. service + .. '/aws4_request' +end + +local function get_signed_headers() + return 'host;x-amz-content-sha256;x-amz-date' +end + +local function get_sha256_digest(s) + local h = resty_sha256:new() + h:update(s or '') + return str.to_hex(h:final()) +end + +local function get_hashed_canonical_request(timestamp, host, uri) + local digest = get_sha256_digest(ngx.var.request_body) + local canonical_request = ngx.var.request_method .. '\n' + .. uri .. '\n' + .. '\n' + .. 'host:' .. host .. '\n' + .. 'x-amz-content-sha256:' .. digest .. '\n' + .. 'x-amz-date:' .. get_iso8601_basic(timestamp) .. '\n' + .. '\n' + .. get_signed_headers() .. '\n' + .. digest + return get_sha256_digest(canonical_request) +end + +local function get_string_to_sign(timestamp, region, service, host, uri) + return 'AWS4-HMAC-SHA256\n' + .. get_iso8601_basic(timestamp) .. '\n' + .. get_cred_scope(timestamp, region, service) .. '\n' + .. get_hashed_canonical_request(timestamp, host, uri) +end + +local function get_signature(derived_signing_key, string_to_sign) + local h = resty_hmac:new(derived_signing_key, resty_hmac.ALGOS.SHA256) + h:update(string_to_sign) + return h:final(nil, true) +end + +local function get_authorization(keys, timestamp, region, service, host, uri) + local derived_signing_key = get_derived_signing_key(keys, timestamp, region, service) + local string_to_sign = get_string_to_sign(timestamp, region, service, host, uri) + local auth = 'AWS4-HMAC-SHA256 ' + .. 'Credential=' .. keys['access_key'] .. '/' .. get_cred_scope(timestamp, region, service) + .. ', SignedHeaders=' .. get_signed_headers() + .. ', Signature=' .. get_signature(derived_signing_key, string_to_sign) + return auth +end + +local function get_service_and_region(host) + local patterns = { + {'s3.amazonaws.com', 's3', 'us-east-1'}, + {'s3-external-1.amazonaws.com', 's3', 'us-east-1'}, + {'s3%-([a-z0-9-]+)%.amazonaws%.com', 's3', nil} + } + + for i,data in ipairs(patterns) do + local region = host:match(data[1]) + if region ~= nil and data[3] == nil then + return data[2], region + elseif region ~= nil then + return data[2], data[3] + end + end + + return 's3', 'auto' +end + +function _M.aws_set_headers(host, uri) + local creds = get_credentials() + local timestamp = tonumber(ngx.time()) + local service, region = get_service_and_region(host) + local auth = get_authorization(creds, timestamp, region, service, host, uri) + + ngx.req.set_header('Authorization', auth) + ngx.req.set_header('Host', host) + ngx.req.set_header('x-amz-date', get_iso8601_basic(timestamp)) +end + +function _M.s3_set_headers(host, uri) + _M.aws_set_headers(host, uri) + ngx.req.set_header('x-amz-content-sha256', get_sha256_digest(ngx.var.request_body)) +end + +return _M diff --git a/docker/web/nginx.conf b/docker/web/nginx.conf index 59b80bc7..788c9ee5 100644 --- a/docker/web/nginx.conf +++ b/docker/web/nginx.conf @@ -2,18 +2,47 @@ upstream philomena { server app:4000 fail_timeout=0; } -upstream s3 { - server files:80 fail_timeout=0; +map $uri $custom_content_type { + default "text/html"; + ~(.*\.png)$ "image/png"; + ~(.*\.jpe?g)$ "image/jpeg"; + ~(.*\.gif)$ "image/gif"; + ~(.*\.svg)$ "image/svg+xml"; + ~(.*\.mp4)$ "video/mp4"; + ~(.*\.webm)$ "video/webm"; } -map $uri $custom_content_type { - default "text/html"; - ~(.*\.png)$ "image/png"; - ~(.*\.jpe?g)$ "image/jpeg"; - ~(.*\.gif)$ "image/gif"; - ~(.*\.svg)$ "image/svg+xml"; - ~(.*\.mp4)$ "video/mp4"; - ~(.*\.webm)$ "video/webm"; +lua_package_path '/etc/nginx/lua/?.lua;;'; +resolver 1.1.1.1 ipv6=off; + +init_by_lua_block { + aws_sig = require('aws-signature') + + function clear_request() + -- Get rid of any client state that could cause + -- issues for the proxied request + for h, _ in pairs(ngx.req.get_headers()) do + if string.lower(h) ~= 'range' then + ngx.req.clear_header(h) + end + end + + ngx.req.set_uri_args({}) + ngx.req.discard_body() + end + + function sign_aws_request() + -- The API token used should not allow writing, but + -- sanitize this anyway to stop an upstream error + if ngx.req.get_method() ~= 'GET' then + ngx.status = ngx.HTTP_UNAUTHORIZED + ngx.say('Unauthorized') + return ngx.exit(ngx.HTTP_UNAUTHORIZED) + end + + clear_request() + aws_sig.s3_set_headers("$S3_HOST", ngx.var.uri) + end } server { @@ -25,61 +54,93 @@ server { client_max_body_size 125000000; client_body_buffer_size 128k; - location ~ ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ { - expires max; - add_header Cache-Control public; - proxy_pass http://s3/$BUCKET_NAME/images/$1/$2/full.$3; - proxy_hide_header Content-Type; - add_header Content-Type $custom_content_type; + location ~ ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ { + rewrite ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ "/$S3_BUCKET/images/$1/$2/full.$3" break; + access_by_lua "sign_aws_request()"; + proxy_pass "$S3_SCHEME://$S3_HOST:$S3_PORT"; + proxy_hide_header Content-Type; + proxy_ssl_server_name on; + + expires max; + add_header Cache-Control public; + add_header Content-Type $custom_content_type; } - location ~ ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ { - add_header Content-Disposition "attachment"; - expires max; - add_header Cache-Control public; - proxy_pass http://s3/$BUCKET_NAME/images/$1/$2/full.$3; - proxy_hide_header Content-Type; - add_header Content-Type $custom_content_type; + location ~ ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ { + rewrite ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ "/$S3_BUCKET/images/$1/$2/full.$3" break; + access_by_lua "sign_aws_request()"; + proxy_pass "$S3_SCHEME://$S3_HOST:$S3_PORT"; + proxy_hide_header Content-Type; + proxy_ssl_server_name on; + + expires max; + add_header Cache-Control public; + add_header Content-Type $custom_content_type; + add_header Content-Disposition "attachment"; } - location ~ ^/img/(.+) { - expires max; - add_header Cache-Control public; - proxy_pass http://s3/$BUCKET_NAME/images/$1; - proxy_hide_header Content-Type; - add_header Content-Type $custom_content_type; + location ~ ^/img/(.+)$ { + rewrite ^/img/(.+)$ "/$S3_BUCKET/images/$1" break; + access_by_lua "sign_aws_request()"; + proxy_pass "$S3_SCHEME://$S3_HOST:$S3_PORT"; + proxy_hide_header Content-Type; + proxy_ssl_server_name on; + + expires max; + add_header Cache-Control public; + add_header Content-Type $custom_content_type; } - location ~ ^/spns/(.+) { - expires max; - add_header Cache-Control public; - proxy_pass http://s3/$BUCKET_NAME/adverts/$1; - proxy_hide_header Content-Type; - add_header Content-Type $custom_content_type; + location ~ ^/spns/(.+) { + rewrite ^/spns/(.+)$ "/$S3_BUCKET/adverts/$1" break; + access_by_lua "sign_aws_request()"; + proxy_pass "$S3_SCHEME://$S3_HOST:$S3_PORT"; + proxy_hide_header Content-Type; + proxy_ssl_server_name on; + + expires max; + add_header Cache-Control public; + add_header Content-Type $custom_content_type; } - location ~ ^/avatars/(.+) { - expires max; - add_header Cache-Control public; - proxy_pass http://s3/$BUCKET_NAME/avatars/$1; - proxy_hide_header Content-Type; - add_header Content-Type $custom_content_type; + location ~ ^/avatars/(.+) { + rewrite ^/avatars/(.+)$ "/$S3_BUCKET/avatars/$1" break; + access_by_lua "sign_aws_request()"; + proxy_pass "$S3_SCHEME://$S3_HOST:$S3_PORT"; + proxy_hide_header Content-Type; + proxy_ssl_server_name on; + + expires max; + add_header Cache-Control public; + add_header Content-Type $custom_content_type; } - location ~ ^/badge-img/(.+) { - expires max; - add_header Cache-Control public; - proxy_pass http://s3/$BUCKET_NAME/badges/$1; - proxy_hide_header Content-Type; - add_header Content-Type $custom_content_type; + # The following two location blocks use an -img suffix to avoid + # conflicting with the application routes. In production, this + # is not necessary since assets will be on a distinct domain. + + location ~ ^/badge-img/(.+) { + rewrite ^/badge-img/(.+)$ "/$S3_BUCKET/badges/$1" break; + access_by_lua "sign_aws_request()"; + proxy_pass "$S3_SCHEME://$S3_HOST:$S3_PORT"; + proxy_hide_header Content-Type; + proxy_ssl_server_name on; + + expires max; + add_header Cache-Control public; + add_header Content-Type $custom_content_type; } - location ~ ^/tag-img/(.+) { - expires max; - add_header Cache-Control public; - proxy_pass http://s3/$BUCKET_NAME/tags/$1; - proxy_hide_header Content-Type; - add_header Content-Type $custom_content_type; + location ~ ^/tag-img/(.+) { + rewrite ^/tag-img/(.+)$ "/$S3_BUCKET/tags/$1" break; + access_by_lua "sign_aws_request()"; + proxy_pass "$S3_SCHEME://$S3_HOST:$S3_PORT"; + proxy_hide_header Content-Type; + proxy_ssl_server_name on; + + expires max; + add_header Cache-Control public; + add_header Content-Type $custom_content_type; } location / {