Improve planeheat script runtime and misc stuff (#85)

* add stderr redirection to all services add error trapping to a bunch of important shell scripts for easier debuging * speed up planeheat.sh * add --compressed to a bunch of curl invocations * .gitignore .swp files * make planefence run at a fixed interval regardless of runtime * add shell script for building the current state of the directory * make sure html directory exists before copying (for dev) * improve shell script debugging message to show full path * fix planeheat for bug introduced in speed improvement for it
2025-12-22 11:17:06 +00:00 · 2021-06-05 15:47:24 +02:00
parent 2ac3ddeb6d
commit aa2726527a
16 changed files with 78 additions and 17 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@

 ### Linux ###
 *~
+.swp

 # temporary files which can be created if a process still has a handle open of a deleted file
 .fuse_hidden*
--- a/4
+++ b/4
@@ -83,7 +83,7 @@ git config --global advice.detachedHead false && \
    chmod a+x /usr/share/planefence/*.sh /usr/share/planefence/*.py /usr/share/planefence/*.pl /etc/services.d/planefence/run && \
    ln -s /usr/share/socket30003/socket30003.cfg /usr/share/planefence/socket30003.cfg && \
    ln -s /usr/share/planefence/config_tweeting.sh /root/config_tweeting.sh && \
-    curl -s -L -o /usr/share/planefence/airlinecodes.txt https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt && \
+    curl --compressed -s -L -o /usr/share/planefence/airlinecodes.txt https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt && \
    echo "main_($(git ls-remote https://github.com/kx1t/docker-planefence HEAD | awk '{ print substr($1,1,7)}'))_$(date +%y-%m-%d-%T%Z)" > /root/.buildtime && \
 #
 # Ensure the planefence and plane-alert config is available for lighttpd:
@@ -95,7 +95,7 @@ git config --global advice.detachedHead false && \
    echo "alias nano=\"nano -l\"" >> /root/.bashrc && \
 #
 # install S6 Overlay
-    curl -s https://raw.githubusercontent.com/mikenye/deploy-s6-overlay/master/deploy-s6-overlay.sh | sh && \
+    curl --compressed -s https://raw.githubusercontent.com/mikenye/deploy-s6-overlay/master/deploy-s6-overlay.sh | sh && \
 #
 # Clean up
    TEMP_PACKAGES="$(</tmp/vars.tmp)" && \
--- a/build-here.sh
+++ b/build-here.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+set -x
+set -e
+trap 'echo -e "[ERROR] $0 in line $LINENO when executing: $BASH_COMMAND"' ERR
+
+# rebuild the container
+mv rootfs/usr/share/planefence/airlinecodes.txt /tmp
+curl --compressed -s -L -o rootfs/usr/share/planefence/airlinecodes.txt https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt
+
+# make the build certs root_certs folder:
+# Note that this is normally done as part of the github actions - we don't have those here, so we need to do it ourselves before building:
+#ls -la /etc/ssl/certs/
+mkdir -p ./root_certs/etc/ssl/certs
+mkdir -p ./root_certs/usr/share/ca-certificates/mozilla
+cp --no-dereference /etc/ssl/certs/*.crt ./root_certs/etc/ssl/certs
+cp --no-dereference /etc/ssl/certs/*.pem ./root_certs/etc/ssl/certs
+cp --no-dereference /usr/share/ca-certificates/mozilla/*.crt ./root_certs/usr/share/ca-certificates/mozilla
+
+echo "$(git branch --show-current)_($(git rev-parse --short HEAD))_$(date +%y-%m-%d-%T%Z)" > rootfs/usr/share/planefence/branch
+
+docker build . -t planefence
+mv /tmp/airlinecodes.txt rootfs/usr/share/planefence/
+rm -f rootfs/usr/share/planefence/branch
+rm -rf ./root_certs
--- a/buildnow.sh
+++ b/buildnow.sh
@@ -12,7 +12,7 @@ pushd ~/git/docker-planefence
 git checkout $BRANCH || exit 2
 git pull
 mv rootfs/usr/share/planefence/airlinecodes.txt /tmp
-curl -s -L -o rootfs/usr/share/planefence/airlinecodes.txt https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt
+curl --compressed -s -L -o rootfs/usr/share/planefence/airlinecodes.txt https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt

 # make the build certs root_certs folder:
 # Note that this is normally done as part of the github actions - we don't have those here, so we need to do it ourselves before building:
--- a/rootfs/etc/services.d/cleanup/run
+++ b/rootfs/etc/services.d/cleanup/run
@@ -1,6 +1,12 @@
 #!/usr/bin/with-contenv bash
 #shellcheck shell=bash

+# redirect stderr to stdout so it's picked up in the docker logs
+exec 2>&1
+# all errors will show a line number and the command used to produce the error
+SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)$(basename "$0")"
+trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR
+
 APPNAME="$(hostname)/cleanup"

 [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Cleanup started as an s6 service"
@@ -159,8 +165,8 @@ GET_AIRLINE_DB ()
 {
    [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Updating airline names database..."
    rm -f /tmp/airlinecodes.txt /tmp/airlines.csv /tmp/airlinecodes.txt.tmp
-    curl -s -L -f https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt -o /tmp/airlinecodes.txt && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got kx1t/planefence-airlinecodes"
-    curl -s -L -f https://raw.githubusercontent.com/jbroutier/whatisflying-db/master/data/airlines.csv -o /tmp/airlines.csv && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got jbroutier/whatisflying-db"
+    curl --compressed -s -L -f https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt -o /tmp/airlinecodes.txt && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got kx1t/planefence-airlinecodes"
+    curl --compressed -s -L -f https://raw.githubusercontent.com/jbroutier/whatisflying-db/master/data/airlines.csv -o /tmp/airlines.csv && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got jbroutier/whatisflying-db"

    #convert JBroutier's DB into our format:
    if [[ -f /tmp/airlines.csv ]]
--- a/rootfs/etc/services.d/get-pa-alertlist/get-pa-alertlist.sh
+++ b/rootfs/etc/services.d/get-pa-alertlist/get-pa-alertlist.sh
@@ -25,7 +25,7 @@ do
 	if [[ "${ALERT:0:5}" == "http:" ]] || [[ "${ALERT:0:6}" == "https:" ]]
 	then
 		# it's a URL and we need to CURL it
-		if [[ "$(curl -L -s --fail -o /tmp/alertlist-$i.txt "$ALERT" ; echo $?)" == "0" ]]
+		if [[ "$(curl --compressed -L -s --fail -o /tmp/alertlist-$i.txt "$ALERT" ; echo $?)" == "0" ]]
 		then
 			[[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] ALERTLIST $ALERT ($i) retrieval succeeded"
 			((i++))
--- a/rootfs/etc/services.d/get-pa-alertlist/run
+++ b/rootfs/etc/services.d/get-pa-alertlist/run
@@ -1,6 +1,12 @@
 #!/usr/bin/with-contenv bash
 #shellcheck shell=bash

+# redirect stderr to stdout so it's picked up in the docker logs
+exec 2>&1
+# all errors will show a line number and the command used to produce the error
+SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)$(basename "$0")"
+trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR
+
 APPNAME="$(hostname)/get-pa-alertlist"

 [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Get-PA-Alertlist started as an s6 service"
--- a/rootfs/etc/services.d/lighttpd/run
+++ b/rootfs/etc/services.d/lighttpd/run
@@ -1,6 +1,12 @@
 #!/usr/bin/with-contenv bash
 #shellcheck shell=bash

+# redirect stderr to stdout so it's picked up in the docker logs
+exec 2>&1
+# all errors will show a line number and the command used to produce the error
+SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)$(basename "$0")"
+trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR
+
 APPNAME="$(hostname)/lighttpd"
 APPPATH="/usr/sbin/lighttpd"
 CONFIGPATH="/etc/lighttpd/lighttpd.conf"
--- a/rootfs/etc/services.d/planefence/run
+++ b/rootfs/etc/services.d/planefence/run
@@ -1,6 +1,12 @@
 #!/usr/bin/with-contenv bash
 #shellcheck shell=bash

+# redirect stderr to stdout so it's picked up in the docker logs
+exec 2>&1
+# all errors will show a line number and the command used to produce the error
+SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)$(basename "$0")"
+trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR
+
 APPNAME="$(hostname)/planefence"
 LOOPTIME=$(grep "PF_INTERVAL" /usr/share/planefence/persist/planefence.config | awk -F "=" '{ print $2 }')
 PLANEFENCEDIR=/usr/share/planefence
@@ -39,6 +45,7 @@ sleep $LOOPTIME
 while true
 do
    LOOPTIME=$(grep "PF_INTERVAL" /usr/share/planefence/persist/planefence.config | awk -F "=" '{ print $2 }')
+	sleep $LOOPTIME &
 	if [[ $(find /run/socket30003/ -name dump1090-*-$(date +%y%m%d).txt) ]]
        then
              starttime=$(date +%s)
@@ -46,13 +53,13 @@ do
              [[ "$TIMING" != "" ]] && export BASETIME=$(date +%s.%2N) || unset BASETIME
              $PLANEFENCEDIR/planefence.sh
              endtime=$(date +%s)
-              [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] PlaneFence ran for $((endtime - starttime)) secs and will be running again at $(date -d "+$LOOPTIME seconds" +"%Y/%m/%d %H:%M:%S")."
+              [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] PlaneFence ran for $((endtime - starttime)) secs and will be running again at $(date -d @$(( $starttime + $LOOPTIME )) +"%Y/%m/%d %H:%M:%S")."
        else
              echo "[$APPNAME][$(date)] /run/socket30003/dump1090-*-(date +%y%m%d).txt not found."
              echo "[$APPNAME][$(date)] If this continues to happen after 5-10 minutes, check this:"
              echo "[$APPNAME][$(date)] Is \"socket30003\" running? Is your feeder producing data?"
    fi
 	# [ -z "$TESTTIME" ] && LOOPTIME="$TESTTIME" # debug code
-	sleep $LOOPTIME
+    wait
 	# echo "[$APPNAME][$(date)] PlaneFence process running..."
 done
--- a/rootfs/etc/services.d/socket30003/run
+++ b/rootfs/etc/services.d/socket30003/run
@@ -1,6 +1,12 @@
 #!/usr/bin/with-contenv bash
 #shellcheck shell=bash

+# redirect stderr to stdout so it's picked up in the docker logs
+exec 2>&1
+# all errors will show a line number and the command used to produce the error
+SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)$(basename "$0")"
+trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR
+
 APPNAME="$(hostname)/socket30003"

 [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Socket30003 started as an s6 service"
--- a/rootfs/usr/share/plane-alert/plane-alert.sh
+++ b/rootfs/usr/share/plane-alert/plane-alert.sh
@@ -37,6 +37,8 @@ PLANEALERTDIR=/usr/share/plane-alert # the directory where this file and planefe
 #
 #

+# all errors will show a line number and the command used to produce the error
+trap 'echo -e "[ERROR] $(basename $0) in line $LINENO when executing: $BASH_COMMAND"' ERR

 function cleanup
 {
--- a/rootfs/usr/share/planefence/airlinename.sh
+++ b/rootfs/usr/share/planefence/airlinename.sh
@@ -117,7 +117,7 @@ echo $a | grep -e '^[A-Za-z]\{3\}[0-9][A-Za-z0-9]*' >/dev/null && b="$(awk -F ',
 # Nothing? Then do an FAA DB lookup
 if [[ "$b" == "" ]] && [[ "${a:0:1}" == "N" ]]
 then
-        b="$(timeout 3 curl -s https://registry.faa.gov/AircraftInquiry/Search/NNumberResult?nNumberTxt=$a | grep 'data-label=\"Name\"'|head -1 | sed 's|.*>\(.*\)<.*|\1|g')"
+        b="$(timeout 3 curl --compressed -s https://registry.faa.gov/AircraftInquiry/Search/NNumberResult?nNumberTxt=$a | grep 'data-label=\"Name\"'|head -1 | sed 's|.*>\(.*\)<.*|\1|g')"
        # If we got something, make sure it will get added to the cache:
        [[ "$b" != "" ]] && MUSTCACHE=1
        [[ "$b" != "" ]] && [[ "$q" == "" ]] && q="faa"
--- a/rootfs/usr/share/planefence/airlinename.sh.org
+++ b/rootfs/usr/share/planefence/airlinename.sh.org
@@ -118,7 +118,7 @@ fi
 # Nothing? Then do an FAA DB lookup
 if [[ "$b" == "" ]] && [[ "${a:0:1}" == "N" ]]
 then
-	b="$(timeout 3 curl -s https://registry.faa.gov/AircraftInquiry/Search/NNumberResult?nNumberTxt=$a | grep 'data-label=\"Name\"'|head -1 | sed 's|.*>\(.*\)<.*|\1|g')"
+	b="$(timeout 3 curl --compressed -s https://registry.faa.gov/AircraftInquiry/Search/NNumberResult?nNumberTxt=$a | grep 'data-label=\"Name\"'|head -1 | sed 's|.*>\(.*\)<.*|\1|g')"
 	# If we got something, make sure it will get added to the cache:
 	[[ "$b" != "" ]] && MUSTCACHE=1
 fi
--- a/rootfs/usr/share/planefence/planefence.sh
+++ b/rootfs/usr/share/planefence/planefence.sh
@@ -28,7 +28,8 @@
 # -----------------------------------------------------------------------------------
 # Only change the variables below if you know what you are doing.

-trap 'echo -e "[ERROR] Line $LINENO when executing: $BASH_COMMAND"' ERR
+# all errors will show a line number and the command used to produce the error
+trap 'echo -e "[ERROR] $(basename $0) in line $LINENO when executing: $BASH_COMMAND"' ERR

 # We need to define the directory where the config file is located:

@@ -572,8 +573,7 @@ tail --lines=+$READLINES $LOGFILEBASE"$FENCEDATE".txt > $INFILETMP

 # First, run planefence.py to create the CSV file:
 LOG "Invoking planefence.py..."
-$PLANEFENCEDIR/planefence.py --logfile=$INFILETMP --outfile=$OUTFILETMP --maxalt=$MAXALT --altcorr=$ALTCORR --dist=$DIST --distunit=$DISTUNIT --lat=$LAT --lon=$LON $VERBOSE $CALCDIST --trackservice=$TRACKSERVICE 2>&1 | LOG
-# without ALTCORR: $PLANEFENCEDIR/planefence.py --logfile=$INFILETMP --outfile=$OUTFILETMP --maxalt=$MAXALT --dist=$DIST --distunit=$DISTUNIT --lat=$LAT --lon=$LON $VERBOSE $CALCDIST --trackservice=$TRACKSERVICE 2>&1 | LOG
+$PLANEFENCEDIR/planefence.py --logfile=$INFILETMP --outfile=$OUTFILETMP --maxalt=$MAXALT --altcorr=$ALTCORR --dist=$DIST --distunit=$DISTUNIT --lat=$LAT --lon=$LON $VERBOSE $CALCDIST --trackservice=$TRACKSERVICE | LOG
 LOG "Returned from planefence.py..."

 # Now we need to combine any double entries. This happens when a plane was in range during two consecutive Planefence runs
--- a/rootfs/usr/share/planefence/planeheat.sh
+++ b/rootfs/usr/share/planefence/planeheat.sh
@@ -180,10 +180,11 @@ tail --lines=+"$((LASTLINE + 1))" "$INFILESOCK" > "$INFILESOCK".tmp
 # Now let's iterate through the entries in the file
 if [[ -f "$INFILECSV" ]]
 then
+    # Now clean the line from any control characters (like stray \r's) and read the line into an array:
+    INPUT=$(tr -d -c '[:print:]\n' <"$INFILECSV")
    while read -r CSVLINE
    do
-        # Now clean the line from any control characters (like stray \r's) and read the line into an array:
-        IFS="," read -r -aRECORD <<< "$(echo -n $CSVLINE | tr -d '[:cntrl:]')"
+        IFS="," read -r -aRECORD <<< "$CSVLINE"
        (( COUNTER++ ))
        LOG "Processing ${RECORD[0]} (${RECORD[2]:11:8} - ${RECORD[3]:11:8}) with COUNTER=$COUNTER, NUMRECORD=${#RECORD[@]}, LASTFENCE=$LASTFENCE"

@@ -206,7 +207,7 @@ then
        else
            LOG "(${RECORD[0]} was previously processed.)"
        fi
-    done < "$INFILECSV"
+    done <<< "$INPUT"
 fi

 # rewrite the latest to $TMPVARS
--- a/rootfs/usr/share/planefence/prep-planefence.sh
+++ b/rootfs/usr/share/planefence/prep-planefence.sh
@@ -39,6 +39,7 @@ fi
 # this cannot be done at build time because the directory is exposed and it is
 # overwritten by the host at start of runtime

+mkdir -p /usr/share/planefence/html
 cp -n /usr/share/planefence/stage/* /usr/share/planefence/html
 rm -f /usr/share/planefence/html/planefence.config
 [[ ! -f /usr/share/planefence/persist/pf-background.jpg ]] && cp -f /usr/share/planefence/html/background.jpg /usr/share/planefence/persist/pf_background.jpg
@@ -228,7 +229,7 @@ if [[ ! -f /usr/share/planefence/persist/plane-alert-db.txt ]] && [[ "$PF_PLANEA
 then
 		echo "[$APPNAME][$(date)] Cannot find or create the plane-alert-db.txt file. Disabling Plane-Alert."
 		echo "[$APPNAME][$(date)] Do this on the host to get a base file:"
-		echo "[$APPNAME][$(date)] curl -s https://raw.githubusercontent.com/kx1t/docker-planefence/plane-alert/plane-alert-db.txt >~/.planefence/plane-alert-db.txt"
+		echo "[$APPNAME][$(date)] curl --compressed -s https://raw.githubusercontent.com/kx1t/docker-planefence/plane-alert/plane-alert-db.txt >~/.planefence/plane-alert-db.txt"
 		echo "[$APPNAME][$(date)] and then restart this docker container"
 		PF_PLANEALERT="OFF"
 fi