Old upgrade code (specific to previous releases) removed and new upgrade procedure adjusted to HAF requirements.

This commit is contained in:
Bartek Wrona 2022-11-21 23:38:43 +01:00
parent d745f8eeb6
commit 0e62052c6c
4 changed files with 46 additions and 234 deletions

View File

@ -3,11 +3,12 @@
set -e
set -o pipefail
echo "Usage ./db_upgrade.sh <user-name> <db-name>"
echo "Usage ./db_upgrade.sh <postgresql_url>"
rm -f ./upgrade.log
for sql in upgrade/assert_public_schema.sql \
postgres_handle_view_changes.sql \
#upgrade/assert_public_schema.sql \
for sql in postgres_handle_view_changes.sql \
upgrade/upgrade_table_schema.sql \
utility_functions.sql \
hive_accounts_view.sql \
@ -78,12 +79,12 @@ for sql in upgrade/assert_public_schema.sql \
update_table_statistics.sql # Must be last
do
echo Executing psql -U $1 -d $2 -f $sql
time psql -a -1 -v "ON_ERROR_STOP=1" -U $1 -d $2 -c '\timing' -f $sql 2>&1 | tee -a -i upgrade.log
echo Executing psql "$1" -f $sql
time psql -a -1 -v "ON_ERROR_STOP=1" "$1" -c '\timing' -f $sql 2>&1 | tee -a -i upgrade.log
echo $?
done
time psql -a -v "ON_ERROR_STOP=1" -U $1 -d $2 -c '\timing' -f upgrade/upgrade_runtime_migration.sql 2>&1 | tee -a -i upgrade.log
time psql -a -v "ON_ERROR_STOP=1" "$1" -c '\timing' -f upgrade/upgrade_runtime_migration.sql 2>&1 | tee -a -i upgrade.log
time psql -a -v "ON_ERROR_STOP=1" -U $1 -d $2 -c '\timing' -f upgrade/do_conditional_vacuum.sql 2>&1 | tee -a -i upgrade.log
time psql -a -v "ON_ERROR_STOP=1" "$1" -c '\timing' -f upgrade/do_conditional_vacuum.sql 2>&1 | tee -a -i upgrade.log

View File

@ -1,6 +1,6 @@
START TRANSACTION;
insert into hive_db_patch_level
insert into hivemind_app.hive_db_patch_level
(level, patch_date, patched_to_revision)
select ds.level, ds.patch_date, ds.patch_revision
from
@ -10,6 +10,6 @@ values
,(34, now(), '9d2cc15bea71a39139abdf49569e0eac6dd0b970') -- https://gitlab.syncad.com/hive/hivemind/-/merge_requests/575
) ds (level, patch_date, patch_revision)
where not exists (select null from hive_db_patch_level hpl where hpl.patched_to_revision = ds.patch_revision);
where not exists (select null from hivemind_app.hive_db_patch_level hpl where hpl.patched_to_revision = ds.patch_revision);
COMMIT;

View File

@ -1,179 +1,4 @@
--- Put runtime data migration code here
START TRANSACTION;
DO
$BODY$
BEGIN
SET work_mem='2GB';
IF EXISTS(SELECT * FROM hivemind_app.hive_db_data_migration WHERE migration = 'Reputation calculation') THEN
RAISE NOTICE 'Performing initial account reputation calculation...';
PERFORM hivemind_app.update_account_reputations(NULL, NULL, True);
ELSE
RAISE NOTICE 'Skipping initial account reputation calculation...';
END IF;
END
$BODY$;
COMMIT;
START TRANSACTION;
DO
$BODY$
BEGIN
IF EXISTS(SELECT * FROM hivemind_app.hive_db_data_migration WHERE migration = 'hive_posts_api_helper fill') THEN
RAISE NOTICE 'Performing initial hivemind_app.hive_posts_api_helper collection...';
SET work_mem='2GB';
TRUNCATE TABLE hivemind_app.hive_posts_api_helper;
DROP INDEX IF EXISTS hivemind_app.hive_posts_api_helper_author_permlink_idx;
DROP INDEX IF EXISTS hivemind_app.hive_posts_api_helper_author_s_permlink_idx;
PERFORM hivemind_app.update_hive_posts_api_helper(NULL, NULL);
CREATE INDEX IF NOT EXISTS hive_posts_api_helper_author_s_permlink_idx ON hivemind_app.hive_posts_api_helper (author_s_permlink);
ELSE
RAISE NOTICE 'Skipping initial hivemind_app.hive_posts_api_helper collection...';
END IF;
END
$BODY$;
COMMIT;
START TRANSACTION;
DO
$BODY$
BEGIN
IF EXISTS(SELECT * FROM hivemind_app.hive_db_data_migration WHERE migration = 'hive_mentions fill') THEN
RAISE NOTICE 'Performing initial post body mentions collection...';
SET work_mem='2GB';
DROP INDEX IF EXISTS hivemind_app.hive_mentions_block_num_idx;
PERFORM hivemind_app.update_hive_posts_mentions(0, (SELECT hb.num FROM hivemind_app.hive_blocks hb ORDER BY hb.num DESC LIMIT 1) );
CREATE INDEX IF NOT EXISTS hive_mentions_block_num_idx ON hivemind_app.hive_mentions (block_num);
ELSE
RAISE NOTICE 'Skipping initial post body mentions collection...';
END IF;
END
$BODY$;
COMMIT;
START TRANSACTION;
DO
$BODY$
BEGIN
IF EXISTS hivemind_app.(SELECT * FROM hivemind_app.hive_db_data_migration WHERE migration = 'update_posts_rshares( 0, head_block_number) execution') THEN
RAISE NOTICE 'Performing posts rshares, hot and trend recalculation on range ( 0, head_block_number)...';
SET work_mem='2GB';
PERFORM hivemind_app.update_posts_rshares(0, (SELECT hb.num FROM hivemind_app.hive_blocks hb ORDER BY hb.num DESC LIMIT 1) );
DELETE FROM hivemind_app.hive_db_data_migration WHERE migration = 'update_posts_rshares( 0, head_block_number) execution';
ELSE
RAISE NOTICE 'Skipping update_posts_rshares( 0, head_block_number) recalculation...';
END IF;
END
$BODY$;
COMMIT;
START TRANSACTION;
DO
$BODY$
BEGIN
IF EXISTS hivemind_app.(SELECT * FROM hivemind_app.hive_db_data_migration WHERE migration = 'update_hive_posts_children_count execution') THEN
RAISE NOTICE 'Performing initial post children count execution ( 0, head_block_number)...';
SET work_mem='2GB';
update hivemind_app.hive_posts set children = 0 where children != 0;
PERFORM hivemind_app.update_all_hive_posts_children_count();
DELETE FROM hivemind_app.hive_db_data_migration WHERE migration = 'update_hive_posts_children_count execution';
ELSE
RAISE NOTICE 'Skipping initial post children count execution ( 0, head_block_number) recalculation...';
END IF;
END
$BODY$;
COMMIT;
START TRANSACTION;
DO
$BODY$
BEGIN
IF EXISTS hivemind_app.(SELECT * FROM hivemind_app.hive_db_data_migration WHERE migration = 'update_hive_post_mentions refill execution') THEN
RAISE NOTICE 'Performing hivemind_app.hive_mentions refill...';
SET work_mem='2GB';
TRUNCATE TABLE hivemind_app.hive_mentions RESTART IDENTITY;
PERFORM hivemind_app.update_hive_posts_mentions(0, (select max(num) from hivemind_app.hive_blocks));
DELETE FROM hivemind_app.hive_db_data_migration WHERE migration = 'update_hive_post_mentions refill execution';
ELSE
RAISE NOTICE 'Skipping hivemind_app.hive_mentions refill...';
END IF;
END
$BODY$;
COMMIT;
START TRANSACTION;
DO
$BODY$
BEGIN
-- Also covers previous changes at a80c7642a1f3b08997af7e8a9915c13d34b7f0e0
-- Also covers previous changes at b100db27f37dda3c869c2756d99ab2856f7da9f9
-- Also covers previous changes at bd83414409b7624e2413b97a62fa7d97d83edd86
IF NOT EXISTS (SELECT * FROM hivemind_app.hive_db_patch_level WHERE patched_to_revision = '1cc9981679157e4e54e5e4a74cca1feb5d49296d')
THEN
RAISE NOTICE 'Performing notification cache initial fill...';
SET work_mem='2GB';
PERFORM hivemind_app.update_notification_cache(NULL, NULL, False);
DELETE FROM hivemind_app.hive_db_data_migration WHERE migration = 'Notification cache initial fill';
ELSE
RAISE NOTICE 'Skipping notification cache initial fill...';
END IF;
END
$BODY$;
COMMIT;
START TRANSACTION;
DO
$BODY$
BEGIN
SET work_mem='2GB';
IF NOT EXISTS(SELECT * FROM hivemind_app.hive_db_patch_level WHERE patched_to_revision = 'cce7fe54a2242b7a80354ee7e50e5b3275a2b039') THEN
RAISE NOTICE 'Performing reputation livesync recalculation...';
--- reputations have to be recalculated from scratch.
UPDATE hivemind_app.hive_accounts SET reputation = 0, is_implicit = True;
PERFORM hivemind_app.update_account_reputations(NULL, NULL, True);
INSERT INTO hivemind_app.hive_db_vacuum_needed
(vacuum_needed)
values
(True)
;
ELSE
RAISE NOTICE 'Skipping reputation livesync recalculation...';
END IF;
END
$BODY$;
COMMIT;
START TRANSACTION;
DO
$BODY$
BEGIN
SET work_mem='2GB';
IF NOT EXISTS(SELECT * FROM hivemind_app.hive_db_patch_level WHERE patched_to_revision = '33dd5e52673335284c6aa28ee89a069f83bd2dc6') THEN
RAISE NOTICE 'Performing reputation data cleanup...';
PERFORM hivemind_app.truncate_account_reputation_data('30 days'::interval);
INSERT INTO hivemind_app.hive_db_vacuum_needed
(vacuum_needed)
values
(True)
;
ELSE
RAISE NOTICE 'Skipping reputation data cleanup...';
END IF;
END
$BODY$;
COMMIT;
--- Must be at the end
TRUNCATE TABLE hivemind_app.hive_db_data_migration;

View File

@ -1,45 +1,31 @@
do $$
BEGIN
ASSERT EXISTS (SELECT * FROM pg_extension WHERE extname='intarray'), 'The database requires created "intarray" extension';
ASSERT EXISTS (SELECT * FROM pg_extension WHERE extname='intarray'), 'The database requires created "intarray" extension';
ASSERT (SELECT setting FROM pg_settings where name='join_collapse_limit' and source='database')::int = 16, 'Bad optimizer settings, use setup_db.sh script to setup target database correctly';
ASSERT (SELECT setting FROM pg_settings where name='from_collapse_limit' and source='database')::int = 16, 'Bad optimizer settings, use setup_db.sh script to setup target database correctly';
ASSERT (SELECT setting FROM pg_settings where name='jit' and source='database')::BOOLEAN = False, 'Bad optimizer settings, use setup_db.sh script to setup target database correctly';
END$$;
CREATE TABLE IF NOT EXISTS hive_db_patch_level
CREATE TABLE IF NOT EXISTS hivemind_app.hive_db_patch_level
(
level SERIAL NOT NULL PRIMARY KEY,
patch_date timestamp without time zone NOT NULL,
patched_to_revision TEXT
);
CREATE TABLE IF NOT EXISTS hive_db_data_migration
CREATE TABLE IF NOT EXISTS hivemind_app.hive_db_data_migration
(
migration varchar(128) not null
);
CREATE TABLE IF NOT EXISTS hive_db_vacuum_needed
CREATE TABLE IF NOT EXISTS hivemind_app.hive_db_vacuum_needed
(
vacuum_needed BOOLEAN NOT NULL
);
TRUNCATE TABLE hive_db_vacuum_needed;
TRUNCATE TABLE hivemind_app.hive_db_vacuum_needed;
DO $$
DECLARE
__version INT;
BEGIN
SELECT CURRENT_SETTING('server_version_num')::INT INTO __version;
EXECUTE 'ALTER DATABASE '||current_database()||' SET join_collapse_limit TO 16';
EXECUTE 'ALTER DATABASE '||current_database()||' SET from_collapse_limit TO 16';
IF __version >= 120000 THEN
RAISE NOTICE 'Disabling a JIT optimization on the current database level...';
EXECUTE 'ALTER DATABASE '||current_database()||' SET jit TO False';
END IF;
END
$$;
SHOW join_collapse_limit;
SHOW from_collapse_limit;
--- Put schema upgrade code here.
--- ####################################### 1.26 release upgrades #######################################
@ -47,10 +33,10 @@ SHOW from_collapse_limit;
--- Changes done in index hive_posts_community_id_id_idx overwritted by MR 575 (see below)
DROP INDEX IF EXISTS hive_posts_community_id_is_pinned_idx;
DROP INDEX IF EXISTS hivemind_app.hive_posts_community_id_is_pinned_idx;
CREATE INDEX IF NOT EXISTS hive_posts_community_id_is_pinned_idx
ON public.hive_posts USING btree
CREATE INDEX IF NOT EXISTS hivemind_app.hive_posts_community_id_is_pinned_idx
ON hivemind_app.hive_posts USING btree
(community_id ASC NULLS LAST)
INCLUDE(id)
WHERE is_pinned AND counter_deleted = 0;
@ -59,63 +45,63 @@ CREATE INDEX IF NOT EXISTS hive_posts_community_id_is_pinned_idx
--- Begin of MR https://gitlab.syncad.com/hive/hivemind/-/merge_requests/575 ---
DROP INDEX IF EXISTS hive_posts_community_id_id_idx;
DROP INDEX IF EXISTS hivemind_app.hive_posts_community_id_id_idx;
CREATE INDEX IF NOT EXISTS hive_posts_community_id_id_idx
ON public.hive_posts USING btree
CREATE INDEX IF NOT EXISTS hivemind_app.hive_posts_community_id_id_idx
ON hivemind_app.hive_posts USING btree
(community_id ASC NULLS LAST, id DESC)
WHERE counter_deleted = 0
;
--- dedicated to bridge_get_ranked_post_by_created_for_community
CREATE INDEX IF NOT EXISTS hive_posts_community_id_not_is_pinned_idx
ON public.hive_posts USING btree
CREATE INDEX IF NOT EXISTS hivemind_app.hive_posts_community_id_not_is_pinned_idx
ON hivemind_app.hive_posts USING btree
(community_id, id DESC)
WHERE NOT is_pinned and depth = 0 and counter_deleted = 0
;
--- Specific to bridge_get_ranked_post_by_trends_for_community
CREATE INDEX IF NOT EXISTS hive_posts_community_id_not_is_paidout_idx
ON public.hive_posts USING btree
CREATE INDEX IF NOT EXISTS hivemind_app.hive_posts_community_id_not_is_paidout_idx
ON hivemind_app.hive_posts USING btree
(community_id)
INCLUDE (id)
WHERE NOT is_paidout AND depth = 0 AND counter_deleted = 0
;
DROP INDEX IF EXISTS hive_posts_author_id_id_idx;
DROP INDEX IF EXISTS hivemind_app.hive_posts_author_id_id_idx;
CREATE INDEX IF NOT EXISTS hive_posts_author_id_id_idx
ON public.hive_posts USING btree
CREATE INDEX IF NOT EXISTS hivemind_app.hive_posts_author_id_id_idx
ON hivemind_app.hive_posts USING btree
(author_id, id DESC)
WHERE counter_deleted = 0
;
DROP INDEX IF EXISTS hive_follows_following_state_idx;
DROP INDEX IF EXISTS hivemind_app.hive_follows_following_state_idx;
CREATE INDEX IF NOT EXISTS hive_follows_following_state_idx
ON public.hive_follows USING btree
CREATE INDEX IF NOT EXISTS hivemind_app.hive_follows_following_state_idx
ON hivemind_app.hive_follows USING btree
(following, state)
;
DROP INDEX IF EXISTS hive_follows_follower_state_idx;
DROP INDEX IF EXISTS hivemind_app.hive_follows_follower_state_idx;
CREATE INDEX IF NOT EXISTS hive_follows_follower_state_idx
ON public.hive_follows USING btree
CREATE INDEX IF NOT EXISTS hivemind_app.hive_follows_follower_state_idx
ON hivemind_app.hive_follows USING btree
(follower, state)
;
DROP INDEX IF EXISTS hive_follows_follower_following_state_idx;
DROP INDEX IF EXISTS hivemind_app.hive_follows_follower_following_state_idx;
CREATE INDEX IF NOT EXISTS hive_follows_follower_following_state_idx
ON public.hive_follows USING btree
CREATE INDEX IF NOT EXISTS hivemind_app.hive_follows_follower_following_state_idx
ON hivemind_app.hive_follows USING btree
(follower, following, state)
;
DROP INDEX IF EXISTS hive_feed_cache_account_id_created_at_post_id_idx;
DROP INDEX IF EXISTS hivemind_app.hive_feed_cache_account_id_created_at_post_id_idx;
--- Dedicated index to bridge_get_account_posts_by_blog
CREATE INDEX IF NOT EXISTS hive_feed_cache_account_id_created_at_post_id_idx
ON public.hive_feed_cache
CREATE INDEX IF NOT EXISTS hivemind_app.hive_feed_cache_account_id_created_at_post_id_idx
ON hivemind_app.hive_feed_cache
(account_id, created_at DESC, post_id DESC)
;