diff --git a/.gitignore b/.gitignore index 6c8a2c3..bae8eff 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ ### Linux ### *~ +.swp # temporary files which can be created if a process still has a handle open of a deleted file .fuse_hidden* diff --git a/Dockerfile b/Dockerfile index 7b57636..0eac457 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,7 +83,7 @@ git config --global advice.detachedHead false && \ chmod a+x /usr/share/planefence/*.sh /usr/share/planefence/*.py /usr/share/planefence/*.pl /etc/services.d/planefence/run && \ ln -s /usr/share/socket30003/socket30003.cfg /usr/share/planefence/socket30003.cfg && \ ln -s /usr/share/planefence/config_tweeting.sh /root/config_tweeting.sh && \ - curl -s -L -o /usr/share/planefence/airlinecodes.txt https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt && \ + curl --compressed -s -L -o /usr/share/planefence/airlinecodes.txt https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt && \ echo "main_($(git ls-remote https://github.com/kx1t/docker-planefence HEAD | awk '{ print substr($1,1,7)}'))_$(date +%y-%m-%d-%T%Z)" > /root/.buildtime && \ # # Ensure the planefence and plane-alert config is available for lighttpd: @@ -95,7 +95,7 @@ git config --global advice.detachedHead false && \ echo "alias nano=\"nano -l\"" >> /root/.bashrc && \ # # install S6 Overlay - curl -s https://raw.githubusercontent.com/mikenye/deploy-s6-overlay/master/deploy-s6-overlay.sh | sh && \ + curl --compressed -s https://raw.githubusercontent.com/mikenye/deploy-s6-overlay/master/deploy-s6-overlay.sh | sh && \ # # Clean up TEMP_PACKAGES="$( rootfs/usr/share/planefence/branch + +docker build . -t planefence +mv /tmp/airlinecodes.txt rootfs/usr/share/planefence/ +rm -f rootfs/usr/share/planefence/branch +rm -rf ./root_certs diff --git a/buildnow.sh b/buildnow.sh index 2200833..7e975aa 100755 --- a/buildnow.sh +++ b/buildnow.sh @@ -12,7 +12,7 @@ pushd ~/git/docker-planefence git checkout $BRANCH || exit 2 git pull mv rootfs/usr/share/planefence/airlinecodes.txt /tmp -curl -s -L -o rootfs/usr/share/planefence/airlinecodes.txt https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt +curl --compressed -s -L -o rootfs/usr/share/planefence/airlinecodes.txt https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt # make the build certs root_certs folder: # Note that this is normally done as part of the github actions - we don't have those here, so we need to do it ourselves before building: diff --git a/rootfs/etc/services.d/cleanup/run b/rootfs/etc/services.d/cleanup/run index 4a7b65e..b99fbaf 100755 --- a/rootfs/etc/services.d/cleanup/run +++ b/rootfs/etc/services.d/cleanup/run @@ -1,6 +1,12 @@ #!/usr/bin/with-contenv bash #shellcheck shell=bash +# redirect stderr to stdout so it's picked up in the docker logs +exec 2>&1 +# all errors will show a line number and the command used to produce the error +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)$(basename "$0")" +trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR + APPNAME="$(hostname)/cleanup" [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Cleanup started as an s6 service" @@ -159,8 +165,8 @@ GET_AIRLINE_DB () { [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Updating airline names database..." rm -f /tmp/airlinecodes.txt /tmp/airlines.csv /tmp/airlinecodes.txt.tmp - curl -s -L -f https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt -o /tmp/airlinecodes.txt && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got kx1t/planefence-airlinecodes" - curl -s -L -f https://raw.githubusercontent.com/jbroutier/whatisflying-db/master/data/airlines.csv -o /tmp/airlines.csv && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got jbroutier/whatisflying-db" + curl --compressed -s -L -f https://raw.githubusercontent.com/kx1t/planefence-airlinecodes/main/airlinecodes.txt -o /tmp/airlinecodes.txt && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got kx1t/planefence-airlinecodes" + curl --compressed -s -L -f https://raw.githubusercontent.com/jbroutier/whatisflying-db/master/data/airlines.csv -o /tmp/airlines.csv && [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Got jbroutier/whatisflying-db" #convert JBroutier's DB into our format: if [[ -f /tmp/airlines.csv ]] diff --git a/rootfs/etc/services.d/get-pa-alertlist/get-pa-alertlist.sh b/rootfs/etc/services.d/get-pa-alertlist/get-pa-alertlist.sh index 85da9ce..92fd73c 100755 --- a/rootfs/etc/services.d/get-pa-alertlist/get-pa-alertlist.sh +++ b/rootfs/etc/services.d/get-pa-alertlist/get-pa-alertlist.sh @@ -25,7 +25,7 @@ do if [[ "${ALERT:0:5}" == "http:" ]] || [[ "${ALERT:0:6}" == "https:" ]] then # it's a URL and we need to CURL it - if [[ "$(curl -L -s --fail -o /tmp/alertlist-$i.txt "$ALERT" ; echo $?)" == "0" ]] + if [[ "$(curl --compressed -L -s --fail -o /tmp/alertlist-$i.txt "$ALERT" ; echo $?)" == "0" ]] then [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] ALERTLIST $ALERT ($i) retrieval succeeded" ((i++)) diff --git a/rootfs/etc/services.d/get-pa-alertlist/run b/rootfs/etc/services.d/get-pa-alertlist/run index df64c7f..aaa5540 100755 --- a/rootfs/etc/services.d/get-pa-alertlist/run +++ b/rootfs/etc/services.d/get-pa-alertlist/run @@ -1,6 +1,12 @@ #!/usr/bin/with-contenv bash #shellcheck shell=bash +# redirect stderr to stdout so it's picked up in the docker logs +exec 2>&1 +# all errors will show a line number and the command used to produce the error +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)$(basename "$0")" +trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR + APPNAME="$(hostname)/get-pa-alertlist" [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Get-PA-Alertlist started as an s6 service" diff --git a/rootfs/etc/services.d/lighttpd/run b/rootfs/etc/services.d/lighttpd/run index a439ee0..d351dcf 100755 --- a/rootfs/etc/services.d/lighttpd/run +++ b/rootfs/etc/services.d/lighttpd/run @@ -1,6 +1,12 @@ #!/usr/bin/with-contenv bash #shellcheck shell=bash +# redirect stderr to stdout so it's picked up in the docker logs +exec 2>&1 +# all errors will show a line number and the command used to produce the error +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)$(basename "$0")" +trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR + APPNAME="$(hostname)/lighttpd" APPPATH="/usr/sbin/lighttpd" CONFIGPATH="/etc/lighttpd/lighttpd.conf" diff --git a/rootfs/etc/services.d/planefence/run b/rootfs/etc/services.d/planefence/run index e63714f..447e894 100755 --- a/rootfs/etc/services.d/planefence/run +++ b/rootfs/etc/services.d/planefence/run @@ -1,6 +1,12 @@ #!/usr/bin/with-contenv bash #shellcheck shell=bash +# redirect stderr to stdout so it's picked up in the docker logs +exec 2>&1 +# all errors will show a line number and the command used to produce the error +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)$(basename "$0")" +trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR + APPNAME="$(hostname)/planefence" LOOPTIME=$(grep "PF_INTERVAL" /usr/share/planefence/persist/planefence.config | awk -F "=" '{ print $2 }') PLANEFENCEDIR=/usr/share/planefence @@ -39,6 +45,7 @@ sleep $LOOPTIME while true do LOOPTIME=$(grep "PF_INTERVAL" /usr/share/planefence/persist/planefence.config | awk -F "=" '{ print $2 }') + sleep $LOOPTIME & if [[ $(find /run/socket30003/ -name dump1090-*-$(date +%y%m%d).txt) ]] then starttime=$(date +%s) @@ -46,13 +53,13 @@ do [[ "$TIMING" != "" ]] && export BASETIME=$(date +%s.%2N) || unset BASETIME $PLANEFENCEDIR/planefence.sh endtime=$(date +%s) - [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] PlaneFence ran for $((endtime - starttime)) secs and will be running again at $(date -d "+$LOOPTIME seconds" +"%Y/%m/%d %H:%M:%S")." + [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] PlaneFence ran for $((endtime - starttime)) secs and will be running again at $(date -d @$(( $starttime + $LOOPTIME )) +"%Y/%m/%d %H:%M:%S")." else echo "[$APPNAME][$(date)] /run/socket30003/dump1090-*-(date +%y%m%d).txt not found." echo "[$APPNAME][$(date)] If this continues to happen after 5-10 minutes, check this:" echo "[$APPNAME][$(date)] Is \"socket30003\" running? Is your feeder producing data?" fi # [ -z "$TESTTIME" ] && LOOPTIME="$TESTTIME" # debug code - sleep $LOOPTIME + wait # echo "[$APPNAME][$(date)] PlaneFence process running..." done diff --git a/rootfs/etc/services.d/socket30003/run b/rootfs/etc/services.d/socket30003/run index 4a62c50..1ae685a 100755 --- a/rootfs/etc/services.d/socket30003/run +++ b/rootfs/etc/services.d/socket30003/run @@ -1,6 +1,12 @@ #!/usr/bin/with-contenv bash #shellcheck shell=bash +# redirect stderr to stdout so it's picked up in the docker logs +exec 2>&1 +# all errors will show a line number and the command used to produce the error +SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd)$(basename "$0")" +trap 'echo -e "[ERROR] $SCRIPT_PATH in line $LINENO when executing: $BASH_COMMAND"' ERR + APPNAME="$(hostname)/socket30003" [[ "$LOGLEVEL" != "ERROR" ]] && echo "[$APPNAME][$(date)] Socket30003 started as an s6 service" diff --git a/rootfs/usr/share/plane-alert/plane-alert.sh b/rootfs/usr/share/plane-alert/plane-alert.sh index 2ffb4fe..fd6f784 100755 --- a/rootfs/usr/share/plane-alert/plane-alert.sh +++ b/rootfs/usr/share/plane-alert/plane-alert.sh @@ -37,6 +37,8 @@ PLANEALERTDIR=/usr/share/plane-alert # the directory where this file and planefe # # +# all errors will show a line number and the command used to produce the error +trap 'echo -e "[ERROR] $(basename $0) in line $LINENO when executing: $BASH_COMMAND"' ERR function cleanup { diff --git a/rootfs/usr/share/planefence/airlinename.sh b/rootfs/usr/share/planefence/airlinename.sh index 0c01205..e894357 100755 --- a/rootfs/usr/share/planefence/airlinename.sh +++ b/rootfs/usr/share/planefence/airlinename.sh @@ -117,7 +117,7 @@ echo $a | grep -e '^[A-Za-z]\{3\}[0-9][A-Za-z0-9]*' >/dev/null && b="$(awk -F ', # Nothing? Then do an FAA DB lookup if [[ "$b" == "" ]] && [[ "${a:0:1}" == "N" ]] then - b="$(timeout 3 curl -s https://registry.faa.gov/AircraftInquiry/Search/NNumberResult?nNumberTxt=$a | grep 'data-label=\"Name\"'|head -1 | sed 's|.*>\(.*\)<.*|\1|g')" + b="$(timeout 3 curl --compressed -s https://registry.faa.gov/AircraftInquiry/Search/NNumberResult?nNumberTxt=$a | grep 'data-label=\"Name\"'|head -1 | sed 's|.*>\(.*\)<.*|\1|g')" # If we got something, make sure it will get added to the cache: [[ "$b" != "" ]] && MUSTCACHE=1 [[ "$b" != "" ]] && [[ "$q" == "" ]] && q="faa" diff --git a/rootfs/usr/share/planefence/airlinename.sh.org b/rootfs/usr/share/planefence/airlinename.sh.org index 2452032..588acd2 100755 --- a/rootfs/usr/share/planefence/airlinename.sh.org +++ b/rootfs/usr/share/planefence/airlinename.sh.org @@ -118,7 +118,7 @@ fi # Nothing? Then do an FAA DB lookup if [[ "$b" == "" ]] && [[ "${a:0:1}" == "N" ]] then - b="$(timeout 3 curl -s https://registry.faa.gov/AircraftInquiry/Search/NNumberResult?nNumberTxt=$a | grep 'data-label=\"Name\"'|head -1 | sed 's|.*>\(.*\)<.*|\1|g')" + b="$(timeout 3 curl --compressed -s https://registry.faa.gov/AircraftInquiry/Search/NNumberResult?nNumberTxt=$a | grep 'data-label=\"Name\"'|head -1 | sed 's|.*>\(.*\)<.*|\1|g')" # If we got something, make sure it will get added to the cache: [[ "$b" != "" ]] && MUSTCACHE=1 fi diff --git a/rootfs/usr/share/planefence/planefence.sh b/rootfs/usr/share/planefence/planefence.sh index ab99c46..71e9fa6 100755 --- a/rootfs/usr/share/planefence/planefence.sh +++ b/rootfs/usr/share/planefence/planefence.sh @@ -28,7 +28,8 @@ # ----------------------------------------------------------------------------------- # Only change the variables below if you know what you are doing. -trap 'echo -e "[ERROR] Line $LINENO when executing: $BASH_COMMAND"' ERR +# all errors will show a line number and the command used to produce the error +trap 'echo -e "[ERROR] $(basename $0) in line $LINENO when executing: $BASH_COMMAND"' ERR # We need to define the directory where the config file is located: @@ -572,8 +573,7 @@ tail --lines=+$READLINES $LOGFILEBASE"$FENCEDATE".txt > $INFILETMP # First, run planefence.py to create the CSV file: LOG "Invoking planefence.py..." -$PLANEFENCEDIR/planefence.py --logfile=$INFILETMP --outfile=$OUTFILETMP --maxalt=$MAXALT --altcorr=$ALTCORR --dist=$DIST --distunit=$DISTUNIT --lat=$LAT --lon=$LON $VERBOSE $CALCDIST --trackservice=$TRACKSERVICE 2>&1 | LOG -# without ALTCORR: $PLANEFENCEDIR/planefence.py --logfile=$INFILETMP --outfile=$OUTFILETMP --maxalt=$MAXALT --dist=$DIST --distunit=$DISTUNIT --lat=$LAT --lon=$LON $VERBOSE $CALCDIST --trackservice=$TRACKSERVICE 2>&1 | LOG +$PLANEFENCEDIR/planefence.py --logfile=$INFILETMP --outfile=$OUTFILETMP --maxalt=$MAXALT --altcorr=$ALTCORR --dist=$DIST --distunit=$DISTUNIT --lat=$LAT --lon=$LON $VERBOSE $CALCDIST --trackservice=$TRACKSERVICE | LOG LOG "Returned from planefence.py..." # Now we need to combine any double entries. This happens when a plane was in range during two consecutive Planefence runs diff --git a/rootfs/usr/share/planefence/planeheat.sh b/rootfs/usr/share/planefence/planeheat.sh index 0e93924..4f0e6a3 100755 --- a/rootfs/usr/share/planefence/planeheat.sh +++ b/rootfs/usr/share/planefence/planeheat.sh @@ -180,10 +180,11 @@ tail --lines=+"$((LASTLINE + 1))" "$INFILESOCK" > "$INFILESOCK".tmp # Now let's iterate through the entries in the file if [[ -f "$INFILECSV" ]] then + # Now clean the line from any control characters (like stray \r's) and read the line into an array: + INPUT=$(tr -d -c '[:print:]\n' <"$INFILECSV") while read -r CSVLINE do - # Now clean the line from any control characters (like stray \r's) and read the line into an array: - IFS="," read -r -aRECORD <<< "$(echo -n $CSVLINE | tr -d '[:cntrl:]')" + IFS="," read -r -aRECORD <<< "$CSVLINE" (( COUNTER++ )) LOG "Processing ${RECORD[0]} (${RECORD[2]:11:8} - ${RECORD[3]:11:8}) with COUNTER=$COUNTER, NUMRECORD=${#RECORD[@]}, LASTFENCE=$LASTFENCE" @@ -206,7 +207,7 @@ then else LOG "(${RECORD[0]} was previously processed.)" fi - done < "$INFILECSV" + done <<< "$INPUT" fi # rewrite the latest to $TMPVARS diff --git a/rootfs/usr/share/planefence/prep-planefence.sh b/rootfs/usr/share/planefence/prep-planefence.sh index cba1d27..18b3629 100755 --- a/rootfs/usr/share/planefence/prep-planefence.sh +++ b/rootfs/usr/share/planefence/prep-planefence.sh @@ -39,6 +39,7 @@ fi # this cannot be done at build time because the directory is exposed and it is # overwritten by the host at start of runtime +mkdir -p /usr/share/planefence/html cp -n /usr/share/planefence/stage/* /usr/share/planefence/html rm -f /usr/share/planefence/html/planefence.config [[ ! -f /usr/share/planefence/persist/pf-background.jpg ]] && cp -f /usr/share/planefence/html/background.jpg /usr/share/planefence/persist/pf_background.jpg @@ -228,7 +229,7 @@ if [[ ! -f /usr/share/planefence/persist/plane-alert-db.txt ]] && [[ "$PF_PLANEA then echo "[$APPNAME][$(date)] Cannot find or create the plane-alert-db.txt file. Disabling Plane-Alert." echo "[$APPNAME][$(date)] Do this on the host to get a base file:" - echo "[$APPNAME][$(date)] curl -s https://raw.githubusercontent.com/kx1t/docker-planefence/plane-alert/plane-alert-db.txt >~/.planefence/plane-alert-db.txt" + echo "[$APPNAME][$(date)] curl --compressed -s https://raw.githubusercontent.com/kx1t/docker-planefence/plane-alert/plane-alert-db.txt >~/.planefence/plane-alert-db.txt" echo "[$APPNAME][$(date)] and then restart this docker container" PF_PLANEALERT="OFF" fi