Authenticate requests to remote S3 endpoint

docker-compose.yml

@@ -87,8 +87,17 @@ services:
     build:
       context: .
       dockerfile: ./docker/web/Dockerfile
+      args:
+        - APP_DIR=/srv/philomena
+        - S3_SCHEME=http
+        - S3_HOST=files
+        - S3_PORT=80
+        - S3_BUCKET=philomena
     volumes:
       - .:/srv/philomena
+    environment:
+      - AWS_ACCESS_KEY_ID=local-identity
+      - AWS_SECRET_ACCESS_KEY=local-credential
     logging:
       driver: "none"
     depends_on:

docker/web/Dockerfile

@@ -1,7 +1,15 @@
-FROM nginx:1.21.4-alpine
+FROM openresty/openresty:1.19.9.1-12-alpine
-ENV APP_DIR /srv/philomena
+ARG APP_DIR
-ENV BUCKET_NAME philomena
+ARG S3_SCHEME
+ARG S3_HOST
+ARG S3_PORT
+ARG S3_BUCKET
+
+RUN apk add --no-cache gettext curl perl && opm get jkeys089/lua-resty-hmac=0.06 && mkdir -p /etc/nginx/lua
+COPY docker/web/aws-signature.lua /etc/nginx/lua
 COPY docker/web/nginx.conf /tmp/docker.nginx
-RUN envsubst '$APP_DIR $BUCKET_NAME' < /tmp/docker.nginx > /etc/nginx/conf.d/default.conf
+RUN envsubst '$APP_DIR $S3_SCHEME $S3_HOST $S3_PORT $S3_BUCKET' < /tmp/docker.nginx > /etc/nginx/conf.d/default.conf && \
+    echo 'env AWS_ACCESS_KEY_ID;' >> /usr/local/openresty/nginx/conf/nginx.conf && \
+    echo 'env AWS_SECRET_ACCESS_KEY;' >> /usr/local/openresty/nginx/conf/nginx.conf
 EXPOSE 80
-CMD ["nginx", "-g", "daemon off;"]
+CMD ["openresty", "-g", "daemon off;"]

docker/web/aws-signature.lua

@@ -0,0 +1,149 @@
+--[[
+Copyright 2018 JobTeaser
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+--]]
+
+local cjson = require('cjson')
+local resty_hmac = require('resty.hmac')
+local resty_sha256 = require('resty.sha256')
+local str = require('resty.string')
+
+local _M = { _VERSION = '0.1.2' }
+
+local function get_credentials ()
+  local access_key = os.getenv('AWS_ACCESS_KEY_ID')
+  local secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
+
+  return {
+    access_key = access_key,
+    secret_key = secret_key
+  }
+end
+
+local function get_iso8601_basic(timestamp)
+  return os.date('!%Y%m%dT%H%M%SZ', timestamp)
+end
+
+local function get_iso8601_basic_short(timestamp)
+  return os.date('!%Y%m%d', timestamp)
+end
+
+local function get_derived_signing_key(keys, timestamp, region, service)
+  local h_date = resty_hmac:new('AWS4' .. keys['secret_key'], resty_hmac.ALGOS.SHA256)
+  h_date:update(get_iso8601_basic_short(timestamp))
+  k_date = h_date:final()
+
+  local h_region = resty_hmac:new(k_date, resty_hmac.ALGOS.SHA256)
+  h_region:update(region)
+  k_region = h_region:final()
+
+  local h_service = resty_hmac:new(k_region, resty_hmac.ALGOS.SHA256)
+  h_service:update(service)
+  k_service = h_service:final()
+
+  local h = resty_hmac:new(k_service, resty_hmac.ALGOS.SHA256)
+  h:update('aws4_request')
+  return h:final()
+end
+
+local function get_cred_scope(timestamp, region, service)
+  return get_iso8601_basic_short(timestamp)
+    .. '/' .. region
+    .. '/' .. service
+    .. '/aws4_request'
+end
+
+local function get_signed_headers()
+  return 'host;x-amz-content-sha256;x-amz-date'
+end
+
+local function get_sha256_digest(s)
+  local h = resty_sha256:new()
+  h:update(s or '')
+  return str.to_hex(h:final())
+end
+
+local function get_hashed_canonical_request(timestamp, host, uri)
+  local digest = get_sha256_digest(ngx.var.request_body)
+  local canonical_request = ngx.var.request_method .. '\n'
+    .. uri .. '\n'
+    .. '\n'
+    .. 'host:' .. host .. '\n'
+    .. 'x-amz-content-sha256:' .. digest .. '\n'
+    .. 'x-amz-date:' .. get_iso8601_basic(timestamp) .. '\n'
+    .. '\n'
+    .. get_signed_headers() .. '\n'
+    .. digest
+  return get_sha256_digest(canonical_request)
+end
+
+local function get_string_to_sign(timestamp, region, service, host, uri)
+  return 'AWS4-HMAC-SHA256\n'
+    .. get_iso8601_basic(timestamp) .. '\n'
+    .. get_cred_scope(timestamp, region, service) .. '\n'
+    .. get_hashed_canonical_request(timestamp, host, uri)
+end
+
+local function get_signature(derived_signing_key, string_to_sign)
+  local h = resty_hmac:new(derived_signing_key, resty_hmac.ALGOS.SHA256)
+  h:update(string_to_sign)
+  return h:final(nil, true)
+end
+
+local function get_authorization(keys, timestamp, region, service, host, uri)
+  local derived_signing_key = get_derived_signing_key(keys, timestamp, region, service)
+  local string_to_sign = get_string_to_sign(timestamp, region, service, host, uri)
+  local auth = 'AWS4-HMAC-SHA256 '
+    .. 'Credential=' .. keys['access_key'] .. '/' .. get_cred_scope(timestamp, region, service)
+    .. ', SignedHeaders=' .. get_signed_headers()
+    .. ', Signature=' .. get_signature(derived_signing_key, string_to_sign)
+  return auth
+end
+
+local function get_service_and_region(host)
+  local patterns = {
+    {'s3.amazonaws.com', 's3', 'us-east-1'},
+    {'s3-external-1.amazonaws.com', 's3', 'us-east-1'},
+    {'s3%-([a-z0-9-]+)%.amazonaws%.com', 's3', nil}
+  }
+
+  for i,data in ipairs(patterns) do
+    local region = host:match(data[1])
+    if region ~= nil and data[3] == nil then
+      return data[2], region
+    elseif region ~= nil then
+      return data[2], data[3]
+    end
+  end
+
+  return 's3', 'auto'
+end
+
+function _M.aws_set_headers(host, uri)
+  local creds = get_credentials()
+  local timestamp = tonumber(ngx.time())
+  local service, region = get_service_and_region(host)
+  local auth = get_authorization(creds, timestamp, region, service, host, uri)
+
+  ngx.req.set_header('Authorization', auth)
+  ngx.req.set_header('Host', host)
+  ngx.req.set_header('x-amz-date', get_iso8601_basic(timestamp))
+end
+
+function _M.s3_set_headers(host, uri)
+  _M.aws_set_headers(host, uri)
+  ngx.req.set_header('x-amz-content-sha256', get_sha256_digest(ngx.var.request_body))
+end
+
+return _M

docker/web/nginx.conf

@@ -2,18 +2,47 @@ upstream philomena {
 	server app:4000 fail_timeout=0;
 }
 
-upstream s3 {
+map $uri $custom_content_type {
-	server files:80 fail_timeout=0;
+	default			"text/html";
+	~(.*\.png)$		"image/png";
+	~(.*\.jpe?g)$	"image/jpeg";
+	~(.*\.gif)$		"image/gif";
+	~(.*\.svg)$		"image/svg+xml";
+	~(.*\.mp4)$		"video/mp4";
+	~(.*\.webm)$	"video/webm";
 }
 
-map $uri $custom_content_type {
+lua_package_path '/etc/nginx/lua/?.lua;;';
-	default		"text/html";
+resolver 1.1.1.1 ipv6=off;
-	~(.*\.png)$	"image/png";
+
-	~(.*\.jpe?g)$	"image/jpeg";
+init_by_lua_block {
-	~(.*\.gif)$	"image/gif";
+	aws_sig = require('aws-signature')
-	~(.*\.svg)$	"image/svg+xml";
+
-	~(.*\.mp4)$	"video/mp4";
+	function clear_request()
-	~(.*\.webm)$	"video/webm";
+		-- Get rid of any client state that could cause
+		-- issues for the proxied request
+		for h, _ in pairs(ngx.req.get_headers()) do
+			if string.lower(h) ~= 'range' then
+				ngx.req.clear_header(h)
+			end
+		end
+
+		ngx.req.set_uri_args({})
+		ngx.req.discard_body()
+	end
+
+	function sign_aws_request()
+		-- The API token used should not allow writing, but
+		-- sanitize this anyway to stop an upstream error
+		if ngx.req.get_method() ~= 'GET' then
+			ngx.status = ngx.HTTP_UNAUTHORIZED
+			ngx.say('Unauthorized')
+			return ngx.exit(ngx.HTTP_UNAUTHORIZED)
+		end
+
+		clear_request()
+		aws_sig.s3_set_headers("$S3_HOST", ngx.var.uri)
+	end
 }
 
 server {
@@ -25,61 +54,93 @@ server {
 	client_max_body_size       125000000;
 	client_body_buffer_size    128k;
 
-	location ~ ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
+	location  ~ ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
-		expires max;
+		rewrite ^/img/view/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ "/$S3_BUCKET/images/$1/$2/full.$3" break;
-		add_header      Cache-Control public;
+		access_by_lua	 		"sign_aws_request()";
-		proxy_pass http://s3/$BUCKET_NAME/images/$1/$2/full.$3;
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
-		proxy_hide_header Content-Type;
+		proxy_hide_header		Content-Type;
-		add_header Content-Type $custom_content_type;
+		proxy_ssl_server_name	on;
+
+		expires 	max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
-	location ~ ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
+	location  ~ ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ {
-		add_header Content-Disposition "attachment";
+		rewrite ^/img/download/(.+)/([0-9]+).*\.([A-Za-z0-9]+)$ "/$S3_BUCKET/images/$1/$2/full.$3" break;
-		expires max;
+		access_by_lua	 		"sign_aws_request()";
-		add_header      Cache-Control public;
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
-		proxy_pass http://s3/$BUCKET_NAME/images/$1/$2/full.$3;
+		proxy_hide_header		Content-Type;
-		proxy_hide_header Content-Type;
+		proxy_ssl_server_name	on;
-		add_header Content-Type $custom_content_type;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
+		add_header	Content-Disposition "attachment";
 	}
 
-	location ~ ^/img/(.+) {
+	location  ~ ^/img/(.+)$ {
-		expires max;
+		rewrite ^/img/(.+)$ "/$S3_BUCKET/images/$1" break;
-		add_header Cache-Control public;
+		access_by_lua	 		"sign_aws_request()";
-		proxy_pass http://s3/$BUCKET_NAME/images/$1;
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
-		proxy_hide_header Content-Type;
+		proxy_hide_header		Content-Type;
-		add_header Content-Type $custom_content_type;
+		proxy_ssl_server_name	on;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
-	location ~ ^/spns/(.+) {
+	location  ~ ^/spns/(.+) {
-		expires max;
+		rewrite ^/spns/(.+)$ "/$S3_BUCKET/adverts/$1" break;
-		add_header Cache-Control public;
+		access_by_lua	 		"sign_aws_request()";
-		proxy_pass http://s3/$BUCKET_NAME/adverts/$1;
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
-		proxy_hide_header Content-Type;
+		proxy_hide_header		Content-Type;
-		add_header Content-Type $custom_content_type;
+		proxy_ssl_server_name	on;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
-	location ~ ^/avatars/(.+) {
+	location  ~ ^/avatars/(.+) {
-		expires max;
+		rewrite ^/avatars/(.+)$ "/$S3_BUCKET/avatars/$1" break;
-		add_header Cache-Control public;
+		access_by_lua	 		"sign_aws_request()";
-		proxy_pass http://s3/$BUCKET_NAME/avatars/$1;
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
-		proxy_hide_header Content-Type;
+		proxy_hide_header		Content-Type;
-		add_header Content-Type $custom_content_type;
+		proxy_ssl_server_name	on;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
-	location ~ ^/badge-img/(.+) {
+	# The following two location blocks use an -img suffix to avoid
-		expires max;
+	# conflicting with the application routes. In production, this
-		add_header Cache-Control public;
+	# is not necessary since assets will be on a distinct domain.
-		proxy_pass http://s3/$BUCKET_NAME/badges/$1;
+
-		proxy_hide_header Content-Type;
+	location  ~ ^/badge-img/(.+) {
-		add_header Content-Type $custom_content_type;
+		rewrite ^/badge-img/(.+)$ "/$S3_BUCKET/badges/$1" break;
+		access_by_lua	 		"sign_aws_request()";
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
+		proxy_hide_header		Content-Type;
+		proxy_ssl_server_name	on;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
-	location ~ ^/tag-img/(.+) {
+	location  ~ ^/tag-img/(.+) {
-		expires max;
+		rewrite ^/tag-img/(.+)$ "/$S3_BUCKET/tags/$1" break;
-		add_header Cache-Control public;
+		access_by_lua	 		"sign_aws_request()";
-		proxy_pass http://s3/$BUCKET_NAME/tags/$1;
+		proxy_pass				"$S3_SCHEME://$S3_HOST:$S3_PORT";
-		proxy_hide_header Content-Type;
+		proxy_hide_header		Content-Type;
-		add_header Content-Type $custom_content_type;
+		proxy_ssl_server_name	on;
+
+		expires		max;
+		add_header	Cache-Control public;
+		add_header	Content-Type $custom_content_type;
 	}
 
 	location / {