From 6f731d3d9bb6d3940cfaca571e810f2ddfde7c30 Mon Sep 17 00:00:00 2001 From: Laur Ivan Date: Mon, 15 May 2023 11:42:37 +0200 Subject: [PATCH] feat!: Add importer. Use specific IP for firefly. --- templates/docker-compose.yml.j2 | 34 +++- templates/env.firefly-import.conf.j2 | 241 +++++++++++++++++++++++++++ 2 files changed, 269 insertions(+), 6 deletions(-) create mode 100644 templates/env.firefly-import.conf.j2 diff --git a/templates/docker-compose.yml.j2 b/templates/docker-compose.yml.j2 index 80ae659..784cf0e 100644 --- a/templates/docker-compose.yml.j2 +++ b/templates/docker-compose.yml.j2 @@ -6,23 +6,40 @@ services: restart: unless-stopped volumes: - firefly_iii_upload:/var/www/html/storage/upload - env_file: env.firefly.conf + env_file: "{{ firefly_config_path | expanduser }}/env.firefly.conf" networks: - - firefly_iii + firefly_iii: + ipv4_address: 177.0.0.10 ports: - "{{ firefly_http_port_app }}:8080" depends_on: - db + db: image: mariadb hostname: fireflyiiidb restart: unless-stopped - env_file: env.db.conf + env_file: "{{ firefly_config_path | expanduser }}/env.db.conf" networks: - - firefly_iii + firefly_iii: +# ipv4_address: 177.0.0.4 volumes: - firefly_iii_db:/var/lib/mysql + importer: + image: "{{ firefly_import_image }}" + hostname: importer + restart: unless-stopped + env_file: + - "{{ firefly_config_path | expanduser }}/env.firefly-import.conf" + ports: + - "{{ firefly_http_port_import }}:8080" + networks: + firefly_iii: +# ipv4_address: 177.0.0.5 + depends_on: + - app + cron: # # To make this work, set STATIC_CRON_TOKEN in your .env file or as an environment variable and replace REPLACEME below @@ -32,7 +49,8 @@ services: restart: unless-stopped command: sh -c "echo \"0 3 * * * wget -qO- http://app:8080/api/v1/cron/{{ firefly_static_cron_token }}\" | crontab - && crond -f -L /dev/stdout" networks: - - firefly_iii + firefly_iii: +# ipv4_address: 177.0.0.6 volumes: firefly_iii_upload: @@ -40,4 +58,8 @@ volumes: networks: firefly_iii: - driver: bridge \ No newline at end of file + ipam: + driver: default + config: + - subnet: "177.0.0.0/24" + # gateway: "177.0.0.1" diff --git a/templates/env.firefly-import.conf.j2 b/templates/env.firefly-import.conf.j2 new file mode 100644 index 0000000..d6f7b34 --- /dev/null +++ b/templates/env.firefly-import.conf.j2 @@ -0,0 +1,241 @@ +# Firefly Data Importer (FIDI) configuration file + +# Where is Firefly III? +# +# 1) Make sure you ADD http:// or https:// +# 2) Make sure you REMOVE any trailing slash from the end of the URL. +# 3) In case of Docker, refer to the internal IP of your Firefly III installation. +# +# Setting this value is not mandatory. But it is very useful. +# +# This variable can be set from a file if you append it with _FILE +# +FIREFLY_III_URL=http://app:8080 + +# +# Imagine Firefly III can be reached at "http://172.16.0.2:8082" (internal Docker network or something). +# But you have a fancy URL: "https://personal-finances.bill.microsoft.com/" +# +# In those cases, you can overrule the URL so when the data importer links back to Firefly III, it uses the correct URL. +# +# 1) Make sure you ADD http:// or https:// +# 2) Make sure you REMOVE any trailing slash from the end of the URL. +# +# IF YOU SET THIS VALUE, YOU MUST ALSO SET THE FIREFLY_III_URL +# +# This variable can be set from a file if you append it with _FILE +# +VANITY_URL={{ firefly_app_url }} + +# +# Set your Firefly III Personal Access Token (OAuth) +# You can create a Personal Access Token on the /profile page: +# go to the OAuth tab, then Personal Access Token and "Create token". +# +# - Do not use the "command line token". That's the WRONG one. +# - Do not use "APP_KEY" value from your Firefly III installation. That's the WRONG one. +# +# Setting this value is not mandatory. Instructions will follow if you omit this field. +# +# This variable can be set from a file if you append it with _FILE +# +FIREFLY_III_ACCESS_TOKEN= +#FIREFLY_III_ACCESS_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJhdWQiOiI5IiwianRpIjoiYmNjM2RhYmE4YmJlNmQwOGExOWZmYzhhOGE2OTk3OTE1MTM4OGE4ZDFiMDk4NDgxNzM3YmQwZWQ2NjRmYWU0MjliYWNiNjA5NmMwMWJkNWIiLCJpYXQiOjE2ODM5Mjk2NjMuOTIwNDg2LCJuYmYiOjE2ODM5Mjk2NjMuOTIwNDg4LCJleHAiOjE3MTU1NTIwNjMuODQ5NTI3LCJzdWIiOiIxIiwic2NvcGVzIjpbXX0.kdzrVScXwFwTTKbxPcQdMt_1M8ZrfWo38in0z-GLTUJXqIS8emIhoCdnuPIDeNd_nOTtH11A9Q0U-aA_jPBIu3If-R3qjtxbPgA08VccXBgB-bq1SLO8LF8ZHE3oXn9zrG7_wis99plgGLQYR6cmB4SPkshTwleTmlbI6r8H7pxkX_C6rn7izv1l57l-WA69ct-KxQySuzsMVEiZGqr85NRYr3W1_74At20BD-G0V7IaiIjGCfUC9elAjbGdts2d5KQOI86YD_OaX0X3g2DQRvP9i4_o3xSrE9taUfVoMCBm3yWMF2xIh32n1xDYl6M9wmbjGRF2iho9-JCMfi8izLVRD8I4Q-K0IBYPgfB2Xuo8ZeeMQ9qhc2numWGm8I5Slf-lW63At9Nm8L89xoy57fQcOoEg-DnwvL6v3i4wuGoiEMJwSXwje0akpcv6cXxRWDjzY9ErYfeHNnDFdgpflLFgJcO9lmJo0T_QUL0vt_uytDG1Tke7ue6cVPxOlhhDTCxMSG6FSLSx-4kopFevNVg3uOL7YhuKf1SJZBhcAQDjilvpwkWFhIyj69Ofn-2dgyUKAO_5m8QQ8EgcYuAhqz1RIuMmpe94XxzZys8WyX85DNDlU5Blq_QPcNyVQwDyGoklDJcjv3pK3ygE9CPvKAX1eLXMRpiVYaCtllabFGY + +# +# You can also use a public client ID. This is available in Firefly III 5.4.0-alpha.3 and higher. +# This is a number (1, 2, 3). If you use the client ID, you can leave the access token empty and vice versa. +# +# This value is not mandatory to set. Instructions will follow if you omit this field. +# +# This variable can be set from a file if you append it with _FILE +# +FIREFLY_III_CLIENT_ID= + +# +# Nordigen information. +# The key and ID can be set from a file if you append it with _FILE +# +NORDIGEN_ID= +NORDIGEN_KEY= +NORDIGEN_SANDBOX=false + +# +# Spectre information +# +# The ID and secret can be set from a file if you append it with _FILE +SPECTRE_APP_ID= +SPECTRE_SECRET= + +# +# Use cache. No need to do this. +# +USE_CACHE=false + +# +# If set to true, the data import will not complain about running into duplicates. +# This will give you cleaner import mails if you run regular imports. +# +# This means that the data importer will not import duplicates, but it will not complain about them either. +# +# This setting has no influence on the settings in your configuration(.json). +# +# Of course, if something goes wrong *because* the transaction is a duplicate you will +# NEVER know unless you start digging in your log files. So be careful with this. +# +IGNORE_DUPLICATE_ERRORS=false + +# +# Auto import settings. Due to security constraints, you MUST enable each feature individually. +# You must also set a secret. The secret is used for the web routes. +# +# The auto-import secret must be a string of at least 16 characters. +# Visit this page for inspiration: https://www.random.org/passwords/?num=1&len=16&format=html&rnd=new +# +# Submit it using ?secret=X +# +# This variable can be set from a file if you append it with _FILE +# +AUTO_IMPORT_SECRET= + +# +# Is the /autoimport even endpoint enabled? +# By default it's disabled, and the secret alone will not enable it. +# +CAN_POST_AUTOIMPORT=false + +# +# Is the /autoupload endpoint enabled? +# By default it's disabled, and the secret alone will not enable it. +# +CAN_POST_FILES=false + +# +# Import directory white list. You need to set this before the auto importer will accept a directory to import from. +# +# This variable can be set from a file if you append it with _FILE +# +IMPORT_DIR_ALLOWLIST= + +# +# When you're running Firefly III under a (self-signed) certificate, +# the data importer may have trouble verifying the TLS connection. +# +# You have a few options to make sure the data importer can connect +# to Firefly III: +# - 'true': will verify all certificates. The most secure option and the default. +# - 'file.pem': refer to a file (you must provide it) to your custom root or intermediate certificates. +# - 'false': will verify NO certificates. Not very secure. +VERIFY_TLS_SECURITY=true + +# +# If you want, you can set a directory here where the data importer will look for import configurations. +# This is a separate setting from the /import directory that the auto-import uses. +# Setting this variable isn't necessary. The default value is "storage/configurations". +# +# This variable can be set from a file if you append it with _FILE +# +JSON_CONFIGURATION_DIR= + +# +# Time out when connecting with Firefly III. +# π*10 seconds is usually fine. +# +CONNECTION_TIMEOUT=31.41 + +# The following variables can be useful when debugging the application +APP_ENV=local +APP_DEBUG=true +LOG_CHANNEL=stack + +# Log level. You can set this from least severe to most severe: +# debug, info, notice, warning, error, critical, alert, emergency +# If you set it to debug your logs will grow large, and fast. If you set it to emergency probably +# nothing will get logged, ever. +LOG_LEVEL=debug + +# TRUSTED_PROXIES is a useful variable when using Docker and/or a reverse proxy. +# Set it to ** and reverse proxies work just fine. +TRUSTED_PROXIES=** + +# +# Time zone +# +TZ=Europe/Amsterdam + +# +# Use ASSET_URL when you're running the data importer in a sub-directory. +# +ASSET_URL= + +# +# Email settings. +# The data importer can send you a message with all errors, warnings and messages +# after a successful import. This is disabled by default +# +ENABLE_MAIL_REPORT=false + +# +# Force Firefly III URL to be secure? +# +# +EXPECT_SECURE_URL=false + +# If enabled, define which mailer you want to use. +# Options include: smtp, mailgun, postmark, sendmail, log, array +# Amazon SES is not supported. +# log = drop mails in the logs instead of sending them +# array = debug mailer that does nothing. +MAIL_MAILER= + +# where to send the report? +MAIL_DESTINATION=noreply@example.com + +# other mail settings +# These variables can be set from a file if you append it with _FILE +MAIL_FROM_ADDRESS=noreply@example.com +MAIL_HOST=smtp.mailtrap.io +MAIL_PORT=2525 +MAIL_USERNAME=username +MAIL_PASSWORD=password +MAIL_ENCRYPTION=null + +# Extra settings depending on your mail configuration above. +# These variables can be set from a file if you append it with _FILE +MAILGUN_DOMAIN= +MAILGUN_SECRET= +MAILGUN_ENDPOINT= +POSTMARK_TOKEN= + +# +# You probably won't need to change these settings. +# +BROADCAST_DRIVER=log +CACHE_DRIVER=file +QUEUE_CONNECTION=sync +SESSION_DRIVER=file +SESSION_LIFETIME=120 +IS_EXTERNAL=false + +REDIS_HOST=127.0.0.1 +REDIS_PASSWORD=null +REDIS_PORT=6379 + +# always use quotes +REDIS_DB="0" +REDIS_CACHE_DB="1" + +# The only tracker supported is Matomo. +# This is used on the public instance over at https://data-importer.firefly-iii.org +TRACKER_SITE_ID= +TRACKER_URL= + +APP_NAME=DataImporter + +# +# The APP_URL environment variable is NOT used anywhere. +# Don't bother setting it to fix your reverse proxy problems. It won't help. +# Don't open issues telling me it doesn't help because it's not supposed to. +# Laravel uses this to generate links on the command line, which is a feature the data importer does not use. +# +APP_URL=http://localhost