update schema 2024-05-13 SQL files

DEVELOPMENT.rst

@@ -7,8 +7,8 @@ Development Setup
 
 Clone the repository and setup virtualenv::
 
-    git clone git@github.com:lalinsky/mbdata.git
-    cd mbdata/
+    git clone https://github.com/acoustid/mbslave.git
+    cd mbslave/
     virtualenv -p python3 venv
     source venv/bin/activate
     pip install poetry
@@ -17,15 +17,14 @@ Clone the repository and setup virtualenv::
 Updating SQL files and models
 =============================
 
-Run these scripts to update SQL files and rebuild SQLAlchemy models from them::
+Run these scripts to update SQL files::
 
     ./scripts/update_sql.sh
-    ./scripts/update_models.sh
 
 Release a new version
 =====================
 
-1. Change the version number in ``mbdata/__init__.py``.
+1. Change the version number in ``mbslave/__init__.py``.
 
 2. Add notes to ``CHANGELOG.rst``
 
@@ -37,4 +36,4 @@ Release a new version
 
     rm -rf dist/
     python setup.py sdist
-    twine upload dist/mbdata-*.tar.gz
+    twine upload dist/mbslave-*.tar.gz

README.rst

@@ -82,6 +82,14 @@ When the MusicBrainz database schema changes, the replication will stop working.
 This is usually announced on the `MusicBrainz blog <http://blog.musicbrainz.org/>`__.
 When it happens, you need to upgrade the database.
 
+Release 2024-05-13 (29)
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Run the upgrade scripts::
+
+    mbslave psql -f updates/schema-change/29.all.sql
+    echo 'UPDATE replication_control SET current_schema_sequence = 29;' | mbslave psql
+
 Release 2023-05-22 (28)
 ~~~~~~~~~~~~~~~~~~~~~~~
 

mbslave/sql/CreateConstraints.sql

@@ -41,7 +41,7 @@ ALTER TABLE artist_alias
 ALTER TABLE editor_collection_type
       ADD CONSTRAINT allowed_collection_entity_type CHECK (
           entity_type IN (
-            'area', 'artist', 'event', 'instrument', 'label',
+            'area', 'artist', 'event', 'genre', 'instrument', 'label',
             'place', 'recording', 'release', 'release_group',
             'series', 'work'
           )
@@ -76,7 +76,8 @@ ALTER TABLE instrument_alias
 
 ALTER TABLE label
   ADD CONSTRAINT control_for_whitespace CHECK (controlled_for_whitespace(name)),
-  ADD CONSTRAINT only_non_empty CHECK (name != '');
+  ADD CONSTRAINT only_non_empty CHECK (name != ''),
+  ADD CONSTRAINT label_code_length CHECK (label_code > 0 AND label_code < 1000000);
 
 ALTER TABLE label_alias
   ADD CONSTRAINT control_for_whitespace CHECK (controlled_for_whitespace(name)),

mbslave/sql/CreateFKConstraints.sql

@@ -667,6 +667,16 @@ ALTER TABLE editor_collection_event
    FOREIGN KEY (event)
    REFERENCES event(id);
 
+ALTER TABLE editor_collection_genre
+   ADD CONSTRAINT editor_collection_genre_fk_collection
+   FOREIGN KEY (collection)
+   REFERENCES editor_collection(id);
+
+ALTER TABLE editor_collection_genre
+   ADD CONSTRAINT editor_collection_genre_fk_genre
+   FOREIGN KEY (genre)
+   REFERENCES genre(id);
+
 ALTER TABLE editor_collection_gid_redirect
    ADD CONSTRAINT editor_collection_gid_redirect_fk_new_id
    FOREIGN KEY (new_id)

mbslave/sql/CreateIndexes.sql

@@ -147,7 +147,7 @@ CREATE INDEX edit_data_idx_link_type ON edit_data USING GIN (
                      (data#>>'{link,link_type,id}')::int,
                      (data#>>'{old,link_type,id}')::int,
                      (data#>>'{new,link_type,id}')::int,
-                     (data#>>'{relationship,link_type,id}')::int
+                     (data#>>'{relationship,link,type,id}')::int
                  ], NULL)
 );
 

mbslave/sql/CreatePrimaryKeys.sql

@@ -70,6 +70,7 @@ ALTER TABLE editor_collection_artist ADD CONSTRAINT editor_collection_artist_pke
 ALTER TABLE editor_collection_collaborator ADD CONSTRAINT editor_collection_collaborator_pkey PRIMARY KEY (collection, editor);
 ALTER TABLE editor_collection_deleted_entity ADD CONSTRAINT editor_collection_deleted_entity_pkey PRIMARY KEY (collection, gid);
 ALTER TABLE editor_collection_event ADD CONSTRAINT editor_collection_event_pkey PRIMARY KEY (collection, event);
+ALTER TABLE editor_collection_genre ADD CONSTRAINT editor_collection_genre_pkey PRIMARY KEY (collection, genre);
 ALTER TABLE editor_collection_gid_redirect ADD CONSTRAINT editor_collection_gid_redirect_pkey PRIMARY KEY (gid);
 ALTER TABLE editor_collection_instrument ADD CONSTRAINT editor_collection_instrument_pkey PRIMARY KEY (collection, instrument);
 ALTER TABLE editor_collection_label ADD CONSTRAINT editor_collection_label_pkey PRIMARY KEY (collection, label);

mbslave/sql/CreateTables.sql

@@ -2467,7 +2467,7 @@ CREATE TABLE label ( -- replicate (verbose)
     end_date_year       SMALLINT,
     end_date_month      SMALLINT,
     end_date_day        SMALLINT,
-    label_code          INTEGER CHECK (label_code > 0 AND label_code < 100000),
+    label_code          INTEGER,
     type                INTEGER, -- references label_type.id
     area                INTEGER, -- references area.id
     comment             VARCHAR(255) NOT NULL DEFAULT '',
@@ -2729,7 +2729,6 @@ CREATE TABLE link_type ( -- replicate
     link_phrase         VARCHAR(255) NOT NULL,
     reverse_link_phrase VARCHAR(255) NOT NULL,
     long_link_phrase    VARCHAR(255) NOT NULL,
-    priority            INTEGER NOT NULL DEFAULT 0,
     last_updated        TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
     is_deprecated       BOOLEAN NOT NULL DEFAULT false,
     has_dates           BOOLEAN NOT NULL DEFAULT true,
@@ -2801,6 +2800,14 @@ CREATE TABLE editor_collection_event (
     comment TEXT DEFAULT '' NOT NULL
 );
 
+CREATE TABLE editor_collection_genre (
+    collection INTEGER NOT NULL, -- PK, references editor_collection.id
+    genre INTEGER NOT NULL, -- PK, references genre.id
+    added TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    position INTEGER NOT NULL DEFAULT 0 CHECK (position >= 0),
+    comment TEXT DEFAULT '' NOT NULL
+);
+
 CREATE TABLE editor_collection_instrument (
     collection INTEGER NOT NULL, -- PK, references editor_collection.id
     instrument INTEGER NOT NULL, -- PK, references instrument.id

mbslave/sql/DropFKConstraints.sql

@@ -130,6 +130,8 @@ ALTER TABLE editor_collection_deleted_entity DROP CONSTRAINT IF EXISTS editor_co
 ALTER TABLE editor_collection_deleted_entity DROP CONSTRAINT IF EXISTS editor_collection_deleted_entity_fk_gid;
 ALTER TABLE editor_collection_event DROP CONSTRAINT IF EXISTS editor_collection_event_fk_collection;
 ALTER TABLE editor_collection_event DROP CONSTRAINT IF EXISTS editor_collection_event_fk_event;
+ALTER TABLE editor_collection_genre DROP CONSTRAINT IF EXISTS editor_collection_genre_fk_collection;
+ALTER TABLE editor_collection_genre DROP CONSTRAINT IF EXISTS editor_collection_genre_fk_genre;
 ALTER TABLE editor_collection_gid_redirect DROP CONSTRAINT IF EXISTS editor_collection_gid_redirect_fk_new_id;
 ALTER TABLE editor_collection_instrument DROP CONSTRAINT IF EXISTS editor_collection_instrument_fk_collection;
 ALTER TABLE editor_collection_instrument DROP CONSTRAINT IF EXISTS editor_collection_instrument_fk_instrument;

mbslave/sql/DropPrimaryKeys.sql

@@ -70,6 +70,7 @@ ALTER TABLE editor_collection_artist DROP CONSTRAINT IF EXISTS editor_collection
 ALTER TABLE editor_collection_collaborator DROP CONSTRAINT IF EXISTS editor_collection_collaborator_pkey;
 ALTER TABLE editor_collection_deleted_entity DROP CONSTRAINT IF EXISTS editor_collection_deleted_entity_pkey;
 ALTER TABLE editor_collection_event DROP CONSTRAINT IF EXISTS editor_collection_event_pkey;
+ALTER TABLE editor_collection_genre DROP CONSTRAINT IF EXISTS editor_collection_genre_pkey;
 ALTER TABLE editor_collection_gid_redirect DROP CONSTRAINT IF EXISTS editor_collection_gid_redirect_pkey;
 ALTER TABLE editor_collection_instrument DROP CONSTRAINT IF EXISTS editor_collection_instrument_pkey;
 ALTER TABLE editor_collection_label DROP CONSTRAINT IF EXISTS editor_collection_label_pkey;

mbslave/sql/DropTables.sql

@@ -74,6 +74,7 @@ DROP TABLE editor_collection_artist;
 DROP TABLE editor_collection_collaborator;
 DROP TABLE editor_collection_deleted_entity;
 DROP TABLE editor_collection_event;
+DROP TABLE editor_collection_genre;
 DROP TABLE editor_collection_gid_redirect;
 DROP TABLE editor_collection_instrument;
 DROP TABLE editor_collection_label;

mbslave/sql/TruncateTables.sql

@@ -74,6 +74,7 @@ TRUNCATE TABLE editor_collection_artist RESTART IDENTITY CASCADE;
 TRUNCATE TABLE editor_collection_collaborator RESTART IDENTITY CASCADE;
 TRUNCATE TABLE editor_collection_deleted_entity RESTART IDENTITY CASCADE;
 TRUNCATE TABLE editor_collection_event RESTART IDENTITY CASCADE;
+TRUNCATE TABLE editor_collection_genre RESTART IDENTITY CASCADE;
 TRUNCATE TABLE editor_collection_gid_redirect RESTART IDENTITY CASCADE;
 TRUNCATE TABLE editor_collection_instrument RESTART IDENTITY CASCADE;
 TRUNCATE TABLE editor_collection_label RESTART IDENTITY CASCADE;

mbslave/sql/updates/20220720-mbs-12508.sh

@@ -29,10 +29,10 @@ then
     FTP_DATA_URL='http://ftp.musicbrainz.org/pub/musicbrainz/data'
 
     echo `date`: Retrieving LATEST from FTP
-    LATEST=$(curl "$FTP_DATA_URL"/fullexport/LATEST)
+    LATEST=$(curl -sSL "$FTP_DATA_URL"/fullexport/LATEST)
 
     echo `date`: Retrieving mbdump-derived.tar.bz2 from FTP
-    curl \
+    curl -L \
         -o "$DUMP_FILE" \
         "$FTP_DATA_URL"/fullexport/"$LATEST"/mbdump-derived.tar.bz2
 fi

mbslave/sql/updates/20231005-edit-data-idx-link-type.sql

@@ -0,0 +1,17 @@
+\set ON_ERROR_STOP 1
+
+BEGIN;
+
+DROP INDEX IF EXISTS edit_data_idx_link_type;
+
+CREATE INDEX edit_data_idx_link_type ON edit_data USING GIN (
+    array_remove(ARRAY[
+                     (data#>>'{link_type,id}')::int,
+                     (data#>>'{link,link_type,id}')::int,
+                     (data#>>'{old,link_type,id}')::int,
+                     (data#>>'{new,link_type,id}')::int,
+                     (data#>>'{relationship,link,type,id}')::int
+                 ], NULL)
+);
+
+COMMIT;

mbslave/sql/updates/20240220-mbs-13403.sql

@@ -0,0 +1,7 @@
+\set ON_ERROR_STOP 1
+
+BEGIN;
+
+ALTER TABLE link_type DROP COLUMN priority CASCADE;
+
+COMMIT;

mbslave/sql/updates/20240223-mbs-13421-fks.sql

@@ -0,0 +1,17 @@
+\set ON_ERROR_STOP 1
+BEGIN;
+
+ALTER TABLE editor_collection_type -- Already dropped in mirror update script
+      ADD CONSTRAINT allowed_collection_entity_type CHECK (
+          entity_type IN ('area', 'artist', 'event', 'genre', 'instrument', 'label', 'place', 'recording', 'release', 'release_group', 'series', 'work')
+      );
+
+ALTER TABLE editor_collection_genre
+   ADD CONSTRAINT editor_collection_genre_fk_collection
+     FOREIGN KEY (collection)
+     REFERENCES editor_collection(id),
+   ADD CONSTRAINT editor_collection_genre_fk_genre
+     FOREIGN KEY (genre)
+     REFERENCES genre(id);
+
+COMMIT;

mbslave/sql/updates/20240223-mbs-13421.sql

@@ -0,0 +1,20 @@
+\set ON_ERROR_STOP 1
+BEGIN;
+
+CREATE TABLE editor_collection_genre (
+    collection INTEGER NOT NULL,
+    genre INTEGER NOT NULL,
+    added TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    position INTEGER NOT NULL DEFAULT 0 CHECK (position >= 0),
+    comment TEXT DEFAULT '' NOT NULL
+);
+
+ALTER TABLE editor_collection_genre ADD CONSTRAINT editor_collection_genre_pkey PRIMARY KEY (collection, genre);
+
+ALTER TABLE editor_collection_type
+      DROP CONSTRAINT IF EXISTS allowed_collection_entity_type;
+
+INSERT INTO editor_collection_type (id, name, entity_type, parent, child_order, gid)
+     VALUES (16, 'Genre', 'genre', NULL, 2, generate_uuid_v3('6ba7b8119dad11d180b400c04fd430c8', 'editor_collection_type' || 16));
+
+COMMIT;

mbslave/sql/updates/20240319-mbs-13514-mirror.sql

@@ -0,0 +1,9 @@
+\set ON_ERROR_STOP 1
+
+BEGIN;
+
+ALTER TABLE label DROP CONSTRAINT IF EXISTS label_label_code_check;
+
+ALTER TABLE label DROP CONSTRAINT IF EXISTS label_code_length;
+
+COMMIT;

mbslave/sql/updates/20240319-mbs-13514.sql

@@ -0,0 +1,7 @@
+\set ON_ERROR_STOP 1
+
+BEGIN;
+
+ALTER TABLE label ADD CONSTRAINT label_code_length CHECK (label_code > 0 AND label_code < 1000000);
+
+COMMIT;

mbslave/sql/updates/schema-change/29.all.sql

@@ -0,0 +1,59 @@
+-- Generated by CompileSchemaScripts.pl from:
+-- 20231005-edit-data-idx-link-type.sql
+-- 20240220-mbs-13403.sql
+-- 20240223-mbs-13421.sql
+-- 20240319-mbs-13514-mirror.sql
+\set ON_ERROR_STOP 1
+BEGIN;
+SET search_path = musicbrainz, public;
+SET LOCAL statement_timeout = 0;
+--------------------------------------------------------------------------------
+SELECT '20231005-edit-data-idx-link-type.sql';
+
+
+DROP INDEX IF EXISTS edit_data_idx_link_type;
+
+CREATE INDEX edit_data_idx_link_type ON edit_data USING GIN (
+    array_remove(ARRAY[
+                     (data#>>'{link_type,id}')::int,
+                     (data#>>'{link,link_type,id}')::int,
+                     (data#>>'{old,link_type,id}')::int,
+                     (data#>>'{new,link_type,id}')::int,
+                     (data#>>'{relationship,link,type,id}')::int
+                 ], NULL)
+);
+
+--------------------------------------------------------------------------------
+SELECT '20240220-mbs-13403.sql';
+
+
+ALTER TABLE link_type DROP COLUMN priority CASCADE;
+
+--------------------------------------------------------------------------------
+SELECT '20240223-mbs-13421.sql';
+
+CREATE TABLE editor_collection_genre (
+    collection INTEGER NOT NULL,
+    genre INTEGER NOT NULL,
+    added TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
+    position INTEGER NOT NULL DEFAULT 0 CHECK (position >= 0),
+    comment TEXT DEFAULT '' NOT NULL
+);
+
+ALTER TABLE editor_collection_genre ADD CONSTRAINT editor_collection_genre_pkey PRIMARY KEY (collection, genre);
+
+ALTER TABLE editor_collection_type
+      DROP CONSTRAINT IF EXISTS allowed_collection_entity_type;
+
+INSERT INTO editor_collection_type (id, name, entity_type, parent, child_order, gid)
+     VALUES (16, 'Genre', 'genre', NULL, 2, generate_uuid_v3('6ba7b8119dad11d180b400c04fd430c8', 'editor_collection_type' || 16));
+
+--------------------------------------------------------------------------------
+SELECT '20240319-mbs-13514-mirror.sql';
+
+
+ALTER TABLE label DROP CONSTRAINT IF EXISTS label_label_code_check;
+
+ALTER TABLE label DROP CONSTRAINT IF EXISTS label_code_length;
+
+COMMIT;

mbslave/sql/updates/schema-change/29.master_and_standalone.sql

@@ -0,0 +1,30 @@
+-- Generated by CompileSchemaScripts.pl from:
+-- 20240223-mbs-13421-fks.sql
+-- 20240319-mbs-13514.sql
+\set ON_ERROR_STOP 1
+BEGIN;
+SET search_path = musicbrainz, public;
+SET LOCAL statement_timeout = 0;
+--------------------------------------------------------------------------------
+SELECT '20240223-mbs-13421-fks.sql';
+
+ALTER TABLE editor_collection_type -- Already dropped in mirror update script
+      ADD CONSTRAINT allowed_collection_entity_type CHECK (
+          entity_type IN ('area', 'artist', 'event', 'genre', 'instrument', 'label', 'place', 'recording', 'release', 'release_group', 'series', 'work')
+      );
+
+ALTER TABLE editor_collection_genre
+   ADD CONSTRAINT editor_collection_genre_fk_collection
+     FOREIGN KEY (collection)
+     REFERENCES editor_collection(id),
+   ADD CONSTRAINT editor_collection_genre_fk_genre
+     FOREIGN KEY (genre)
+     REFERENCES genre(id);
+
+--------------------------------------------------------------------------------
+SELECT '20240319-mbs-13514.sql';
+
+
+ALTER TABLE label ADD CONSTRAINT label_code_length CHECK (label_code > 0 AND label_code < 1000000);
+
+COMMIT;