mirror of
https://github.com/PostHog/posthog.git
synced 2024-12-01 12:21:02 +01:00
bbd41ef04a
From https://medium.com/@taylorhughes/three-quick-tips-from-two-years-with-celery-c05ff9d7f9eb > By default, preforking Celery workers distribute tasks to their worker processes as soon as they are received, regardless of whether the process is currently busy with other tasks. > If you have a set of tasks that take varying amounts of time to complete — either deliberately or due to unpredictable network conditions, etc. — this will cause unexpected delays in total execution time for tasks in the queue. This is 100% the case for us. This should "load balance" the tasks better across workers.
83 lines
2.1 KiB
Bash
Executable File
83 lines
2.1 KiB
Bash
Executable File
#!/bin/bash
|
|
set -e
|
|
|
|
help () {
|
|
echo "$0 - start PostHog's Celery worker"
|
|
echo
|
|
echo "$0 [options]"
|
|
echo
|
|
echo "Options:"
|
|
echo " --help, -h show this brief help"
|
|
echo " --with-scheduler start RedBeat, the Celery scheduler (deprecates --with-beat)"
|
|
echo " --concurrency=<N> start N workers (overrides env var WEB_CONCURRENCY)"
|
|
echo
|
|
echo "Advanced Celery options (disabled by default):"
|
|
echo " --with-gossip start Celery gossip (useful for Prometheus)"
|
|
echo " --with-heartbeat start Celery internal heartbeat (normally not useful)"
|
|
echo " --with-mingle start Celery mingle (normally not useful)"
|
|
exit 0
|
|
}
|
|
|
|
with_scheduler=false
|
|
with_gossip=false
|
|
with_heartbeat=false
|
|
with_mingle=false
|
|
|
|
while test $# -gt 0; do
|
|
case "$1" in
|
|
-h|--help)
|
|
help
|
|
;;
|
|
--with-scheduler)
|
|
with_scheduler=true
|
|
shift
|
|
;;
|
|
--with-beat) # Deprecated since the name is too similar to "heartbeat"
|
|
echo "⚠️ Using docker-worker-celery with --with-beat. This argument is deprecated. Use --with-scheduler instead!"
|
|
with_scheduler=true
|
|
shift
|
|
;;
|
|
--with-gossip)
|
|
with_gossip=true
|
|
shift
|
|
;;
|
|
--with-heartbeat)
|
|
with_heartbeat=true
|
|
shift
|
|
;;
|
|
--with-mingle)
|
|
with_mingle=true
|
|
shift
|
|
;;
|
|
--concurrency*)
|
|
export WEB_CONCURRENCY=`echo $1 | sed -e 's/^[^=]*=//g'`
|
|
shift
|
|
;;
|
|
*)
|
|
break
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ "$with_scheduler" == "true" ]; then
|
|
./bin/docker-worker-beat &
|
|
fi
|
|
|
|
FLAGS=()
|
|
FLAGS+=("-Ofair")
|
|
[ "$with_gossip" == "false" ] && FLAGS+=("--without-gossip")
|
|
[ "$with_mingle" == "false" ] && FLAGS+=("--without-mingle")
|
|
[ "$with_heartbeat" == "false" ] && FLAGS+=("--without-heartbeat")
|
|
|
|
# On Heroku $WEB_CONCURRENCY contains suggested number of forks per dyno type
|
|
# https://github.com/heroku/heroku-buildpack-python/blob/main/vendor/WEB_CONCURRENCY.sh
|
|
[[ -n "${WEB_CONCURRENCY}" ]] && FLAGS+=" --concurrency $WEB_CONCURRENCY"
|
|
|
|
echo
|
|
echo "celery -A posthog worker ${FLAGS[*]}"
|
|
echo
|
|
celery -A posthog worker ${FLAGS[*]}
|
|
|
|
# Stop the beat!
|
|
trap 'kill $(jobs -p)' EXIT
|