-
Notifications
You must be signed in to change notification settings - Fork 91
/
Dockerfile
147 lines (117 loc) · 4.91 KB
/
Dockerfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
FROM node:16-alpine AS base
# Build the dashboard directoy, only what is needed.
FROM base AS deps
# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
RUN apk add --no-cache libc6-compat
WORKDIR /app/dashboard
COPY /dashboard/package.json /dashboard/package-lock.json* ./
RUN npm i
# Rebuild the source code only when needed
FROM base AS builder
WORKDIR /app
COPY --from=deps /app/dashboard/node_modules ./dashboard/node_modules
COPY ./dashboard ./dashboard
ENV NEXT_TELEMETRY_DISABLED 1
WORKDIR /app/dashboard
RUN npm run build
#Production image, copy all the files and run next
FROM base AS runner
WORKDIR /app/dashboard
ENV NODE_ENV production
ENV NEXT_TELEMETRY_DISABLED 1
RUN addgroup --system --gid 1001 nodejs
RUN adduser --system --uid 1001 nextjs
COPY --from=builder /app/dashboard/public ./public
COPY --from=builder --chown=nextjs:nodejs /app/dashboard/.next/standalone ./
COPY --from=builder --chown=nextjs:nodejs /app/dashboard/.next/static ./.next/static
COPY --from=builder --chown=nextjs:nodejs /app/dashboard/out ./out
# Build Go services
FROM golang:1.21 as go-builder
RUN apt update && \
apt install -y protobuf-compiler
RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
WORKDIR /app
COPY go.mod ./
COPY go.sum ./
RUN go mod download
COPY ./filestore/ ./filestore/
COPY ./health/ ./health/
COPY fferr/ fferr/
COPY api/ api/
COPY helpers/ helpers/
COPY lib/ lib/
COPY metadata/ metadata/
COPY metrics/ metrics/
COPY proto/ proto/
COPY coordinator/ coordinator/
COPY provider/ provider/
COPY runner/ runner/
COPY serving/ serving/
COPY types/ types/
COPY kubernetes/ kubernetes/
COPY config/ config/
COPY logging/ logging/
RUN protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative ./proto/serving.proto
RUN protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative ./metadata/proto/metadata.proto
RUN mkdir execs
RUN go build -o execs/api api/main.go
RUN go build -o execs/metadata metadata/server/server.go
RUN go build -o execs/coordinator coordinator/main/main.go
RUN go build -o execs/dashboard_metadata metadata/dashboard/dashboard_metadata.go
RUN go build -o execs/serving serving/main/main.go
# Final image
FROM golang:1.21
WORKDIR /app
# Install and configure Supervisor
RUN apt-get update && apt-get install -y supervisor
RUN mkdir -p /var/lock/apache2 /var/run/apache2 /var/run/sshd /var/log/supervisor
RUN apt-get install -y nginx --option=Dpkg::Options::=--force-confdef
# Install Node 16 for internal dashboard server
RUN curl -sL https://deb.nodesource.com/setup_16.x | sh
RUN apt-get update
RUN apt-get install -y nodejs
# Install MeiliSearch
RUN curl -L https://install.meilisearch.com | sh
# Setup Etcd
RUN git clone -b v3.4.16 https://github.com/etcd-io/etcd.git
WORKDIR /app/etcd
RUN go mod download
RUN ./build
WORKDIR /app
RUN ETCD_UNSUPPORTED_ARCH=arm64 ./etcd/bin/etcd --version
# Setup Spark
ARG SPARK_FILEPATH=/app/provider/scripts/spark/offline_store_spark_runner.py
ARG SPARK_PYTHON_PACKAGES=/app/provider/scripts/spark/python_packages.sh
ARG SPARK_REQUIREMENTS=/app/provider/scripts/spark/requirements.txt
ARG MATERIALIZE_NO_TIMESTAMP_QUERY_PATH=/app/provider/queries/materialize_no_ts.sql
ARG MATERIALIZE_TIMESTAMP_QUERY_PATH=/app/provider/queries/materialize_ts.sql
COPY provider/scripts/spark/offline_store_spark_runner.py $SPARK_FILEPATH
COPY provider/scripts/spark/python_packages.sh $SPARK_PYTHON_PACKAGES
COPY provider/scripts/spark/requirements.txt $SPARK_REQUIREMENTS
COPY provider/queries/materialize_no_ts.sql $MATERIALIZE_NO_TIMESTAMP_QUERY_PATH
COPY provider/queries/materialize_ts.sql $MATERIALIZE_TIMESTAMP_QUERY_PATH
# Take the MD5 hash of the Spark runner script and store it in a file for use by the config package
# when determining the remove filepath in cloud object storage (e.g. S3). By adding the hash as a suffix
# to the file, we ensure that different versions of the script are uploaded to cloud object storage
# without overwriting previous or future versions.
RUN cat /app/provider/scripts/spark/offline_store_spark_runner.py | md5sum \
| awk '{print $1}' \
| xargs echo -n > /app/provider/scripts/spark/offline_store_spark_runner_md5.txt
ENV PYTHON_LOCAL_INIT_PATH=$SPARK_PYTHON_PACKAGES
# Setup Nginx
COPY nginx.conf /etc/nginx/nginx.conf
# Copy built Go services
COPY --from=go-builder /app/execs /app/execs
# Copy built dashboard
COPY --from=runner /app/dashboard ./dashboard
ENV SERVING_PORT="8082"
ENV SERVING_HOST="0.0.0.0"
ENV ETCD_ARCH=""
ENV MEILI_LOG_LEVEL="WARN"
ENV FF_GET_EQUIVALENT_VARIANTS="false"
EXPOSE 7878
EXPOSE 80
COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
HEALTHCHECK --interval=5m --timeout=10s --start-period=10s --retries=3 CMD curl --fail http://localhost/ || exit 1
CMD ["/usr/bin/supervisord"]