View Revisions: Issue #3280

Summary 0003280: SymmetricDS servers fail to start in cluster due to sym schema upgrade race condition
Revision 2017-10-16 16:31 by mmichalek
Description When starting several clustered SymmetricDS servers simultaneously with an upgraded version of the software, several or all of the servers could fail to start due to DDL errors. A viable workaround is when upgrading, allow one server to start up cleanly (and therefore complete the schema upgrade) and then start the other servers in the cluster.

Startup examples include:

2017-10-03 22:05:45,389 WARN [SymmetricDS] [PostgreSqlSymmetricDialect] [symmetric-engine-startup-0] DDL failed: ALTER TABLE sym_node_group_link
    DROP CONSTRAINT sym_fk_lnk_2_grp_src
2017-10-03 22:05:45,390 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist. Failed to execute: ALTER TABLE sym_node_group_link
    DROP CONSTRAINT sym_fk_lnk_2_grp_src
2017-10-03 22:05:45,390 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:2493147748]
org.jumpmind.db.sql.SqlException: ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300)
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493)
    at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571)
    at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244)
    at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530)
Caused by: org.postgresql.util.PSQLException: ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist
    at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476)
    at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189)
    at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300)
    at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301)
    at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491)
    ... 10 more


2017-05-03 22:06:20,945 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists. Failed to execute: ALTER TABLE sym_conflict
    ADD CONSTRAINT sym_fk_cf_2_grp_lnk FOREIGN KEY (source_node_group_id, target_node_group_id) REFERENCES sym_node_group_link (source_node_group_id, target_node_group_id)
2017-10-03 22:06:20,946 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:2197569610]
org.jumpmind.db.sql.SqlException: ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300)
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493)
    at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571)
    at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244)
    at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530)
Caused by: org.postgresql.util.PSQLException: ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists
    at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476)
    at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189)
    at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300)
    at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301)
    at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491)
    ... 10 more


OR

2017-10-03 22:05:45,665 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: column "data_event_action" of relation "sym_channel" already exists. Failed to execute: ALTER TABLE public.sym_channel
         ADD COLUMN data_event_action CHAR(1)
2017-10-03 22:05:45,666 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:4252188107]
org.jumpmind.db.sql.SqlException: ERROR: column "data_event_action" of relation "sym_channel" already exists
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300)
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493)
    at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571)
    at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244)
    at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530)
Caused by: org.postgresql.util.PSQLException: ERROR: column "data_event_action" of relation "sym_channel" already exists
    at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476)
    at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189)
    at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300)
    at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301)
    at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491)
    ... 10 more
Revision 2017-10-16 16:29 by mmichalek
Description When starting several clustered SymmetricDS servers simultaneously with an upgraded version of the software, several or all of the servers could fail to start due to DDL errors such as:

2017-10-03 22:05:45,389 WARN [SymmetricDS] [PostgreSqlSymmetricDialect] [symmetric-engine-startup-0] DDL failed: ALTER TABLE sym_node_group_link
    DROP CONSTRAINT sym_fk_lnk_2_grp_src
2017-10-03 22:05:45,390 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist. Failed to execute: ALTER TABLE sym_node_group_link
    DROP CONSTRAINT sym_fk_lnk_2_grp_src
2017-10-03 22:05:45,390 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:2493147748]
org.jumpmind.db.sql.SqlException: ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300)
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493)
    at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571)
    at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244)
    at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530)
Caused by: org.postgresql.util.PSQLException: ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist
    at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476)
    at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189)
    at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300)
    at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301)
    at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491)
    ... 10 more


2017-05-03 22:06:20,945 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists. Failed to execute: ALTER TABLE sym_conflict
    ADD CONSTRAINT sym_fk_cf_2_grp_lnk FOREIGN KEY (source_node_group_id, target_node_group_id) REFERENCES sym_node_group_link (source_node_group_id, target_node_group_id)
2017-10-03 22:06:20,946 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:2197569610]
org.jumpmind.db.sql.SqlException: ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300)
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493)
    at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571)
    at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244)
    at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530)
Caused by: org.postgresql.util.PSQLException: ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists
    at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476)
    at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189)
    at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300)
    at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301)
    at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491)
    ... 10 more


OR

2017-10-03 22:05:45,665 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: column "data_event_action" of relation "sym_channel" already exists. Failed to execute: ALTER TABLE public.sym_channel
         ADD COLUMN data_event_action CHAR(1)
2017-10-03 22:05:45,666 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:4252188107]
org.jumpmind.db.sql.SqlException: ERROR: column "data_event_action" of relation "sym_channel" already exists
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300)
    at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493)
    at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526)
    at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580)
    at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571)
    at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244)
    at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530)
Caused by: org.postgresql.util.PSQLException: ERROR: column "data_event_action" of relation "sym_channel" already exists
    at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476)
    at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189)
    at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300)
    at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301)
    at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287)
    at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264)
    at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416)
    at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401)
    at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491)
    ... 10 more