96 switch from azure table storage to sqlite #98

Merged
noahspannbauer merged 11 commits from 96-switch-from-azure-table-storage-to-sqlite into main 2025-11-23 11:51:01 -05:00
26 changed files with 119 additions and 504 deletions
Showing only changes of commit 23b0361718 - Show all commits

View File

@@ -1,58 +0,0 @@
name: 'API Build'
on:
workflow_call:
inputs:
environment_name:
required: true
type: string
version_number:
required: true
type: string
jobs:
build:
runs-on: ubuntu-latest
environment: ${{ inputs.environment_name }}
steps:
- name: Checkout
uses: actions/checkout@v4
with:
sparse-checkout: |
api
- name: Install Nest CLI
run: |
npm install -g @nestjs/cli
- name: 'Setup Node'
uses: actions/setup-node@v4
with:
node-version: ${{ vars.NODE_VERSION }}
- name: Install dependencies
run: |
npm ci
- name: Build
run: |
npm run build
- name: Log into Docker Hub
if: ${{ github.event_name != 'pull_request' }}
uses: docker/login-action@v3
with:
username: ${{ vars.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Setup Docker Buildx
if: ${{ github.event_name != 'pull_request' }}
uses: docker/setup-buildx-action@v3
- name: Build and push to registry
if: ${{ github.event_name != 'pull_request' }}
uses: docker/build-push-action@v6
with:
push: true
tags: noahspan/flying-api:${{ inputs.version_number }}
context: .
target: api

View File

@@ -1,70 +0,0 @@
name: 'App Build'
on:
workflow_call:
inputs:
environment_name:
required: true
type: string
version_number:
required: true
type: string
env:
VITE_API_URL: ${{ vars.VITE_API_URL }}
VITE_CLIENT_ID: ${{ vars.VITE_CLIENT_ID }}
VITE_TENANT_ID: ${{ vars.VITE_TENANT_ID }}
VITE_REDIRECT_URL: ${{ vars.VITE_REDIRECT_URL }}
jobs:
build:
runs-on: ubuntu-latest
environment: ${{ inputs.environment_name }}
steps:
- uses: actions/checkout@v4
with:
sparse-checkout: |
app
- uses: pnpm/action-setup@v4
name: Install pnpm
with:
version: 10
run_install: false
- name: 'Setup Node'
uses: actions/setup-node@v4
with:
node-version: ${{ vars.NODE_VERSION }}
- name: Install
run: |
pnpm install
- name: Build
run: |
pnpm --filter app build
- name: Deploy
if: ${{ github.event_name != 'pull_request' }}
run: |
pnpm --filter app --prod deploy ./.prod/app
- name: Setup Docker Buildx
if: ${{ github.event_name != 'pull_request' }}
uses: docker/setup-buildx-action@v3
- name: Log into Docker Hub
if: ${{ github.event_name != 'pull_request' }}
uses: docker/login-action@v3
with:
username: ${{ vars.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push to registry
if: ${{ github.event_name != 'pull_request' }}
uses: docker/build-push-action@v6
with:
push: true
tags: noahspan/flying-app:${{ inputs.version_number }}
context: .
target: app

View File

@@ -8,48 +8,25 @@ jobs:
changes:
uses: ./.github/workflows/changes.yaml
build-api:
if: ${{ needs.changes.outputs.api == 'true' }}
name: build-api
needs:
- changes
uses: ./.github/workflows/api_build.yaml
with:
environment_name: test
version_number: ${{ github.run_id }}
secrets: inherit
deploy-api:
name: deploy-api
needs:
- changes
- build-api
uses: ./.github/workflows/deploy.yaml
with:
app_name: flying-api
environment_name: test
version_number: ${{ github.run_id }}
secrets: inherit
build-app:
build:
if: ${{ needs.changes.outputs.app == 'true' }}
name: build-app
name: build
needs:
- changes
uses: ./.github/workflows/app_build.yaml
uses: ./.github/workflows/build.yaml
with:
environment_name: test
version_number: ${{ github.run_id }}
secrets: inherit
deploy-app:
name: deploy-app
deploy:
name: deploy
needs:
- changes
- build-app
- build
uses: ./.github/workflows/deploy.yaml
with:
app_name: flying-app
app_name: flying
environment_name: test
version_number: ${{ github.run_id }}
secrets: inherit

View File

@@ -5,40 +5,21 @@ on:
- 'v[0-9]+.[0-9]+.[0-9]+'
jobs:
build-api:
name: build-api
uses: ./.github/workflows/api_build.yaml
build:
name: build
uses: ./.github/workflows/build.yaml
with:
environment_name: prod
version_number: ${{ github.ref_name }}
secrets: inherit
deploy-api:
name: deploy-api
deploy:
name: deploy
needs:
- build-api
- build
uses: ./.github/workflows/deploy.yaml
with:
app_name: flying-api
environment_name: prod
version_number: ${{ github.ref_name }}
secrets: inherit
build-app:
name: build-app
uses: ./.github/workflows/app_build.yaml
with:
environment_name: prod
version_number: ${{ github.ref_name }}
secrets: inherit
deploy-app:
name: deploy-app
needs:
- build-app
uses: ./.github/workflows/deploy.yaml
with:
app_name: flying-app
app_name: flying
environment_name: prod
version_number: ${{ github.ref_name }}
secrets: inherit

View File

@@ -1,46 +1,18 @@
# FROM --platform=linux/amd64 node:18-alpine AS base
# ENV PNPM_HOME="/pnpm"
# ENV PATH="$PNPM_HOME:$PATH"
# RUN corepack enable
FROM --platform=linux/amd64 node:22-slim
# FROM base AS api
# COPY /api/dist /app/dist
# COPY /api/node_modules /app/node_modules
# WORKDIR /app
# EXPOSE 3000
# # CMD ["node", "dist/main.js"]
# ENTRYPOINT ["tail", "-f", "/dev/null"]
# FROM base AS app
# COPY /app/dist /app/dist
# WORKDIR /app
# RUN npm i -g serve
# EXPOSE 8080
# CMD [ "serve", "-s", "/dist", "-p", "8080" ]
FROM --platform=linux/amd64 node:22-slim AS base
FROM base AS api
COPY ./api/entrypoint.sh ./entrypoint.sh
RUN chmod +x entrypoint.sh
WORKDIR app
COPY ./api/dist ./api/dist
COPY ./api/package.json package-lock.json ./api
COPY ./client/dist ./client/dist
WORKDIR api
COPY ./api/dist ./dist
COPY ./api/package.json package-lock.json .
RUN npm ci
WORKDIR /
COPY ./api/entrypoint.sh ./entrypoint.sh
RUN chmod +x entrypoint.sh
EXPOSE 3000
ENTRYPOINT ["/entrypoint.sh"]
# ENTRYPOINT ["tail", "-f", "/dev/null"]
FROM base AS app
WORKDIR app
COPY ./app/dist ./dist
RUN npm i -g serve
EXPOSE 8080
CMD [ "serve", "-s", "dist", "-p", "8080" ]
# ENTRYPOINT ["tail", "-f", "/dev/null"]

View File

@@ -1,3 +1,3 @@
#!/bin/bash
npx typeorm migration:run -d ./dist/config/typeorm-cli.config.js
node ./dist/main.js
npx typeorm migration:run -d ./app/api/dist/database/data-source.js
node ./app/api/dist/main.js

View File

@@ -9,8 +9,8 @@ const configService = new ConfigService();
export const dataSourceOptions: DataSourceOptions = {
type: 'better-sqlite3',
database: configService.get<string>('DB_PATH'),
entities: ['dist/**/*.entity.js'],
migrations: ['dist/database/migrations/*.js'],
entities: ['../**/*.entity.js'],
migrations: ['./migrations/*.js'],
synchronize: configService.get<boolean>('DB_SYNC')
}

View File

@@ -41,7 +41,7 @@ export class LogController {
return await this.logService.find(id);
} catch (error) {
const customError = error as CustomError;
console.log(error)
throw new HttpException(customError.message, customError.statusCode);
}
}
@@ -50,10 +50,11 @@ export class LogController {
@Public()
async findAll(): Promise<LogEntity[]> {
try {
console.log('blah')
return await this.logService.findAll();
} catch (error) {
const customError = error as CustomError;
console.log(error)
throw new HttpException(customError.message, customError.statusCode);
}
}

View File

@@ -26,7 +26,7 @@ export class PilotInterceptor implements NestInterceptor {
};
})
}
console.log(req.headers.authorization)
if (req.headers.authorization) {
const authHeader = req.headers.authorization;
const token = authHeader && authHeader.split(' ')[1];

View File

@@ -19,7 +19,7 @@ export default defineConfig({
target: 'http://localhost:3000',
changeOrigin: true,
secure: false,
rewrite: (path) => path.replace(/^\/api/, '')
// rewrite: (path) => path.replace(/^\/api/, '')
}
}
}

View File

@@ -16,23 +16,22 @@ services:
container_name: restore
image: litestream/litestream:0.3.13
volumes:
- ./api/src/database/flying.db:/var/lib/data/flying.db
- ./api/src/database/litestream.yml:/mnt/litestream/litestream.yml
- ./database/flying.db:/var/lib/data/flying.db
- ./database/litestream/litestream.yml:/mnt/litestream/litestream.yml
- backup:/mnt/data/backup
- data:/var/lib/data
command: restore -config /mnt/litestream/litestream.yml -if-db-not-exists -if-replica-exists /var/lib/data/flying.db
api:
container_name: flying-api
app:
container_name: flying
build:
context: .
target: api
ports:
- '3000:3000'
env_file:
- ./api/.env
environment:
- DB_PATH=/var/lib/data/flying.db
- DB_PATH=../../var/lib/data/flying.db
volumes:
- data:/var/lib/data
depends_on:
@@ -43,22 +42,14 @@ services:
container_name: replicate
image: litestream/litestream:0.3.13
volumes:
- ./api/src/database/litestream.yml:/mnt/litestream/litestream.yml
- ./database/litestream/litestream.yml:/mnt/litestream/litestream.yml
- backup:/mnt
- data:/var/lib/data
command: replicate -config /mnt/litestream/litestream.yml
depends_on:
api:
app:
condition: service_started
# app:
# container_name: flying-app
# build:
# context: .
# target: app
# ports:
# - '8080:8080'
volumes:
# azurite-data:
backup:

View File

@@ -1,51 +0,0 @@
locals {
app_name = {
test = "flying-api-test"
prod = "flying-api-prod"
}
container_image = {
test = "docker.io/noahspan/flying-api:v2.0.0-beta"
prod = "docker.io/noahspan/flying-api:v2.0.0-beta"
}
container_name = {
test = "flying-api-test"
prod = "flying-api-prod"
}
custom_domain_count = {
test = 0
prod = 0
}
ingress_external_enabled = {
test = true
prod = true
}
ingress_target_port = {
test = 3000
prod = 3000
}
ingress_transport = {
test = "auto"
prod = "auto"
}
traffic_weight_percentage = {
test = 100
prod = 100
}
template_min_replicas = {
test = 0
prod = 0
}
template_max_replicas = {
test = 2
prod = 2
}
}

View File

@@ -1,39 +0,0 @@
output "app_name" {
value = local.app_name[var.environment]
}
output "container_image" {
value = local.container_image[var.environment]
}
output "container_name" {
value = local.container_name[var.environment]
}
output "custom_domain_count" {
value = local.custom_domain_count[var.environment]
}
output "ingress_external_enabled" {
value = local.ingress_external_enabled[var.environment]
}
output "ingress_target_port" {
value = local.ingress_target_port[var.environment]
}
output "ingress_transport" {
value = local.ingress_transport[var.environment]
}
output "traffic_weight_percentage" {
value = local.traffic_weight_percentage[var.environment]
}
output "template_min_replicas" {
value = local.template_min_replicas[var.environment]
}
output "template_max_replicas" {
value = local.template_max_replicas[var.environment]
}

View File

@@ -1 +0,0 @@
variable "environment" {}

View File

@@ -1,56 +0,0 @@
locals {
app_name = {
test = "flying-test"
prod = "flying-prod"
}
container_app_environment_name = {
test = "noahspan-test"
prod = "noahspan-prod"
}
container_image = {
test = "noahspan/flying-app:v2.0.0-alpha"
prod = "noahspan/flying-app:v2.0.0-alpha"
}
container_name = {
test = "flying-app-test"
prod = "flying-app-prod"
}
custom_domain_count = {
test = 0
prod = 1
}
ingress_external_enabled = {
test = true
prod = true
}
ingress_target_port = {
test = 8080
prod = 8080
}
ingress_transport = {
test = "auto"
prod = "auto"
}
traffic_weight_percentage = {
test = 100
prod = 100
}
template_min_replicas = {
test = 0
prod = 0
}
template_max_replicas = {
test = 2
prod = 2
}
}

View File

@@ -1,43 +0,0 @@
output "app_name" {
value = local.app_name[var.environment]
}
output "container_app_environment_name" {
value = local.container_app_environment_name[var.environment]
}
output "container_image" {
value = local.container_image[var.environment]
}
output "container_name" {
value = local.container_name[var.environment]
}
output "custom_domain_count" {
value = local.custom_domain_count[var.environment]
}
output "ingress_external_enabled" {
value = local.ingress_external_enabled[var.environment]
}
output "ingress_target_port" {
value = local.ingress_target_port[var.environment]
}
output "ingress_transport" {
value = local.ingress_transport[var.environment]
}
output "traffic_weight_percentage" {
value = local.traffic_weight_percentage[var.environment]
}
output "template_min_replicas" {
value = local.template_min_replicas[var.environment]
}
output "template_max_replicas" {
value = local.template_max_replicas[var.environment]
}

View File

@@ -1 +0,0 @@
variable "environment" {}

View File

@@ -1,9 +1,36 @@
module "app" {
source = "./app"
environment = var.environment
}
locals {
app_name = {
test = "flying-test"
prod = "flying-prod"
}
module "storage" {
source = "./storage"
environment = var.environment
}
container_image = {
test = "noahspan/flying-app:v2.0.0-alpha"
prod = "noahspan/flying-app:v2.0.0-alpha"
}
container_name = {
test = "flying-container-test"
prod = "flying-container-prod"
}
container_app_environment_name = {
test = "noahspan-test"
prod = "noahspan-prod"
}
storage_account_name = {
test = "noahspanflyingtest"
prod = "noahspanflyingprod"
}
storage_account_containers = {
test = ["tracks"]
prod = ["tracks"]
}
storage_account_storage_shares = {
test = ["flying-test-database-share"]
prod = ["flying-prod-database-share"]
}
}

View File

@@ -1,7 +1,27 @@
output "app" {
value = module.app
output "app_name" {
value = local.app_name[var.environment]
}
output "storage" {
value = module.storage
output "container_image" {
value = local.container_image[var.environment]
}
output "container_name" {
value = local.container_name[var.environment]
}
output "container_app_environment_name" {
value = local.container_app_environment_name[var.environment]
}
output "storage_account_name" {
value = local.storage_account_name[var.environment]
}
output "storage_account_containers" {
value = local.storage_account_containers[var.environment]
}
output "storage_account_storage_shares" {
value = local.storage_account_storage_shares[var.environment]
}

View File

@@ -1,21 +0,0 @@
locals {
account_name = {
test = "noahspanflyingtest"
prod = "noahspanflyingprod"
}
containers = {
test = ["tracks"]
prod = ["tracks"]
}
shares = {
test = ["flying-test-files"]
prod = ["flying-prod-files"]
}
tables = {
test = ["logs", "pilots"]
prod = ["logs", "pilots"]
}
}

View File

@@ -1,15 +0,0 @@
output "account_name" {
value = local.account_name[var.environment]
}
output "containers" {
value = local.containers[var.environment]
}
output "shares" {
value = local.shares[var.environment]
}
output "tables" {
value = local.tables[var.environment]
}

View File

@@ -1 +0,0 @@
variable "environment" {}

View File

@@ -1,12 +1,12 @@
resource "azurerm_container_app" "container_app" {
name = module.environment.app.app_name
name = module.environment.app_name
container_app_environment_id = data.azurerm_container_app_environment.container_app_environment.id
resource_group_name = data.azurerm_resource_group.resource_group.name
revision_mode = "Single"
template {
min_replicas = module.environment.app.template_min_replicas
max_replicas = module.environment.app.template_max_replicas
min_replicas = 0
max_replicas = 2
init_container {
args = ["restore", "-if-db-not-exists", "-if-replica-exists", "/var/lib/data/flying.db"]
@@ -60,9 +60,9 @@ resource "azurerm_container_app" "container_app" {
container {
cpu = 0.25
image = module.environment.app.container_image
image = "noahspan/flying-app:v2.0.0-alpha"
memory = "0.5Gi"
name = module.environment.app.container_name
name = "flying"
env {
name = "AZURE_STORAGE_CONNECTION_STRING"
@@ -111,7 +111,7 @@ resource "azurerm_container_app" "container_app" {
volume {
name = "backup"
storage_name = azurerm_container_app_environment_storage.container_app_environment_storage_backup.name
storage_name = azurerm_container_app_environment_storage.container_app_environment_storage_database.name
storage_type = "AzureFile"
}
@@ -123,13 +123,13 @@ resource "azurerm_container_app" "container_app" {
ingress {
allow_insecure_connections = false
external_enabled = module.environment.app.ingress_external_enabled
target_port = module.environment.app.ingress_target_port
transport = module.environment.app.ingress_transport
external_enabled = true
target_port = 3000
transport = "auto"
traffic_weight {
latest_revision = true
percentage = module.environment.app.traffic_weight_percentage
percentage = 100
}
}

View File

@@ -1,8 +1,8 @@
resource "azurerm_container_app_environment_storage" "container_app_environment_storage_backup" {
name = "${module.environment.app.app_name}-backup"
resource "azurerm_container_app_environment_storage" "container_app_environment_storage_database" {
name = "${module.environment.app_name}-database"
container_app_environment_id = data.azurerm_container_app_environment.container_app_environment.id
account_name = azurerm_storage_account.storage_account.name
share_name = azurerm_storage_share.backup_storage_share.name
share_name = azurerm_storage_share.storage_share[0].name
access_key = azurerm_storage_account.storage_account.primary_access_key
access_mode = "ReadOnly"
}

View File

@@ -5,6 +5,6 @@ data "azurerm_resource_group" "resource_group" {
}
data "azurerm_container_app_environment" "container_app_environment" {
name = module.environment.app.container_app_environment_name
name = module.environment.container_app_environment_name
resource_group_name = var.RESOURCE_GROUP_NAME
}

View File

@@ -1,5 +1,5 @@
resource "azurerm_storage_account" "storage_account" {
name = module.environment.storage.account_name
name = module.environment.storage_account_name
resource_group_name = data.azurerm_resource_group.resource_group.name
location = data.azurerm_resource_group.resource_group.location
account_tier = "Standard"
@@ -7,13 +7,15 @@ resource "azurerm_storage_account" "storage_account" {
}
resource "azurerm_storage_container" "storage_container" {
name = "tracks"
count = length(module.environment.storage_account_containers)
name = module.environment.storage_account_containers[count.index]
storage_account_id = azurerm_storage_account.storage_account.id
container_access_type = "private"
}
resource "azurerm_storage_share" "backup_storage_share" {
name = "${module.environment.app.app_name}-backup-storage-share"
resource "azurerm_storage_share" "storage_share" {
count = length(module.environment.storage_account_storage_shares)
name = module.environment.storage_account_storage_shares[count.index]
quota = 50
storage_account_name = azurerm_storage_account.storage_account.name
}