Drive/sunetdrive
7
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Use node1 everywhere for statistics

Browse source
This commit is contained in:
Micke Nordin 2023-02-17 10:28:18 +01:00
parent 4c014aaa97
commit edddcbc5da
Signed by untrusted user: Micke
GPG key ID: 0DA0A7A5708FE257
4 changed files with 11 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
templates/script/aggregate.sh
View file

@ -27,21 +27,21 @@ Customer;Total GB;Users;Product"
for customer in $(${yq} -r '.fullnodes | .[]' ${commonyaml}); do for customer in $(${yq} -r '.fullnodes | .[]' ${commonyaml}); do
product=1 # Prisplan 1 product=1 # Prisplan 1
csv="${csv} csv="${csv}
$(rclone cat "${billingbucket}/${customer}-usage/${customer}-latest.csv" | $(rclone cat --no-check-certificate --webdav-headers "Host,sunet.drive.sunet.se" --use-cookies "${billingbucket}/${customer}-usage/${customer}-latest.csv" |
grep -E -v '^DATE|^Customer' | grep -E -v '^DATE|^Customer' |
sed 's/$/;1/')" sed 's/$/;1/')"
done done
for customer in $(${yq} -r '.singlenodes | .[]' ${commonyaml}); do for customer in $(${yq} -r '.singlenodes | .[]' ${commonyaml}); do
product=2 # Prisplan 2 product=2 # Prisplan 2
csv="${csv} csv="${csv}
$(rclone cat "${billingbucket}/${customer}-usage/${customer}-latest.csv" | $(rclone --no-check-certificate --webdav-headers "Host,sunet.drive.sunet.se" --use-cookies cat "${billingbucket}/${customer}-usage/${customer}-latest.csv" |
grep -E -v '^DATE|^Customer' | grep -E -v '^DATE|^Customer' |
sed 's/$/;'${product}'/')" sed 's/$/;'${product}'/')"
done done
echo "${csv}" >"${aggregatefile}" echo "${csv}" >"${aggregatefile}"
rclone copy "${aggregatefile}" "${aggregatedir}/" rclone copy --no-check-certificate --webdav-headers "Host,sunet.drive.sunet.se" --use-cookies "${aggregatefile}" "${aggregatedir}/"
mv "${aggregatefile}" "latest.csv" mv "${aggregatefile}" "latest.csv"
rclone move "latest.csv" "${latestdir}/" rclone move --no-check-certificate --webdav-headers "Host,sunet.drive.sunet.se" --use-cookies "latest.csv" "${latestdir}/"
cd "${olddir}" || (echo "Could not switch back to old dir" && exit 1) cd "${olddir}" || (echo "Could not switch back to old dir" && exit 1)
rmdir "${tempdir}" rmdir "${tempdir}"

2
templates/script/makebuckets.erb.sh
View file

@ -38,7 +38,7 @@ rclone="rclone --config /root/.rclone.conf"
# These are dynamic # These are dynamic
buckets="$(${rclone} lsd "${rcp}:" | awk '{print $NF}')" buckets="$(${rclone} lsd "${rcp}:" | awk '{print $NF}')"
users=$(${rclone} cat "statistics:drive-server-coms/${rcp}/users.json" | jq '. | with_entries( select(.key | match("@") ) )') users=$(${rclone} cat --no-check-certificate --webdav-headers "Host,sunet.drive.sunet.se" --use-cookies "statistics:drive-server-coms/${rcp}/users.json" | jq '. | with_entries( select(.key | match("@") ) )')
for eppn in $(echo "${users}" | jq -r keys[]); do for eppn in $(echo "${users}" | jq -r keys[]); do
user=${eppn%@*} user=${eppn%@*}

2
templates/script/rclone.conf.erb
View file

@ -14,7 +14,7 @@ endpoint = <%= @s3_host_mirror %>
acl = private acl = private
[statistics] [statistics]
type = webdav type = webdav
url = https://sunet.drive.sunet.se/remote.php/dav/files/_script/ url = https://89.46.20.42/remote.php/dav/files/_script/
vendor = nextcloud vendor = nextcloud
user = _script user = _script
pass = <%= @statistics_secret %> pass = <%= @statistics_secret %>

10
templates/script/usage.erb.sh
View file

@ -13,7 +13,7 @@ else
customer="<%= @customer %>" customer="<%= @customer %>"
allowmixedcustomers=no allowmixedcustomers=no
fi fi
userjson=$(rclone cat "statistics:drive-server-coms/${location}/users.json") userjson=$(rclone cat --no-check-certificate --webdav-headers "Host,sunet.drive.sunet.se" --use-cookies "statistics:drive-server-coms/${location}/users.json")
users=$(echo ${userjson} | jq -r '.| keys | .[] | test("^((?!(^admin|^[_])).)*$")' | grep true | wc -l) users=$(echo ${userjson} | jq -r '.| keys | .[] | test("^((?!(^admin|^[_])).)*$")' | grep true | wc -l)
outfile1="${customer}-${date}-detailed.csv" outfile1="${customer}-${date}-detailed.csv"
outfile2="${customer}-${date}.csv" outfile2="${customer}-${date}.csv"
@ -59,11 +59,11 @@ rclone mkdir "${location}:drive-${location}-share"
echo -n "${result1}" >"${outfile1}" echo -n "${result1}" >"${outfile1}"
echo -n "${result2}" >"${outfile2}" echo -n "${result2}" >"${outfile2}"
rclone copyto --use-cookies "${outfile1}" "statistics:drive-storage-report/${customer}-usage/daily/${outfile1}" rclone copyto --no-check-certificate --webdav-headers "Host,sunet.drive.sunet.se" --use-cookies "${outfile1}" "statistics:drive-storage-report/${customer}-usage/daily/${outfile1}"
rclone copyto --use-cookies "${outfile2}" "statistics:drive-storage-report/${customer}-usage/daily/${outfile2}" rclone copyto --no-check-certificate --webdav-headers "Host,sunet.drive.sunet.se" --use-cookies "${outfile2}" "statistics:drive-storage-report/${customer}-usage/daily/${outfile2}"
rclone copyto --use-cookies "${outfile1}" "statistics:drive-storage-report/${customer}-usage/${customer}-latest-detailed.csv" rclone copyto --no-check-certificate --webdav-headers "Host,sunet.drive.sunet.se" --use-cookies "${outfile1}" "statistics:drive-storage-report/${customer}-usage/${customer}-latest-detailed.csv"
rclone copyto --use-cookies "${outfile2}" "statistics:drive-storage-report/${customer}-usage/${customer}-latest.csv" rclone copyto --no-check-certificate --webdav-headers "Host,sunet.drive.sunet.se" --use-cookies "${outfile2}" "statistics:drive-storage-report/${customer}-usage/${customer}-latest.csv"
rm "${outfile1}" rm "${outfile1}"
rm "${outfile2}" rm "${outfile2}"