View Revisions: Issue #3280
Summary | 0003280: SymmetricDS servers fail to start in cluster due to sym schema upgrade race condition | ||
---|---|---|---|
Revision | 2017-10-16 16:31 by mmichalek | ||
Description | When starting several clustered SymmetricDS servers simultaneously with an upgraded version of the software, several or all of the servers could fail to start due to DDL errors. A viable workaround is when upgrading, allow one server to start up cleanly (and therefore complete the schema upgrade) and then start the other servers in the cluster. Startup examples include: 2017-10-03 22:05:45,389 WARN [SymmetricDS] [PostgreSqlSymmetricDialect] [symmetric-engine-startup-0] DDL failed: ALTER TABLE sym_node_group_link DROP CONSTRAINT sym_fk_lnk_2_grp_src 2017-10-03 22:05:45,390 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist. Failed to execute: ALTER TABLE sym_node_group_link DROP CONSTRAINT sym_fk_lnk_2_grp_src 2017-10-03 22:05:45,390 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:2493147748] org.jumpmind.db.sql.SqlException: ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300) at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493) at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157) at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449) at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571) at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244) at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530) Caused by: org.postgresql.util.PSQLException: ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476) at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189) at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300) at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301) at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491) ... 10 more 2017-05-03 22:06:20,945 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists. Failed to execute: ALTER TABLE sym_conflict ADD CONSTRAINT sym_fk_cf_2_grp_lnk FOREIGN KEY (source_node_group_id, target_node_group_id) REFERENCES sym_node_group_link (source_node_group_id, target_node_group_id) 2017-10-03 22:06:20,946 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:2197569610] org.jumpmind.db.sql.SqlException: ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300) at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493) at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157) at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449) at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571) at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244) at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530) Caused by: org.postgresql.util.PSQLException: ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476) at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189) at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300) at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301) at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491) ... 10 more OR 2017-10-03 22:05:45,665 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: column "data_event_action" of relation "sym_channel" already exists. Failed to execute: ALTER TABLE public.sym_channel ADD COLUMN data_event_action CHAR(1) 2017-10-03 22:05:45,666 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:4252188107] org.jumpmind.db.sql.SqlException: ERROR: column "data_event_action" of relation "sym_channel" already exists at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300) at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493) at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157) at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449) at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571) at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244) at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530) Caused by: org.postgresql.util.PSQLException: ERROR: column "data_event_action" of relation "sym_channel" already exists at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476) at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189) at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300) at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301) at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491) ... 10 more |
||
Revision | 2017-10-16 16:29 by mmichalek | ||
Description | When starting several clustered SymmetricDS servers simultaneously with an upgraded version of the software, several or all of the servers could fail to start due to DDL errors such as: 2017-10-03 22:05:45,389 WARN [SymmetricDS] [PostgreSqlSymmetricDialect] [symmetric-engine-startup-0] DDL failed: ALTER TABLE sym_node_group_link DROP CONSTRAINT sym_fk_lnk_2_grp_src 2017-10-03 22:05:45,390 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist. Failed to execute: ALTER TABLE sym_node_group_link DROP CONSTRAINT sym_fk_lnk_2_grp_src 2017-10-03 22:05:45,390 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:2493147748] org.jumpmind.db.sql.SqlException: ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300) at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493) at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157) at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449) at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571) at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244) at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530) Caused by: org.postgresql.util.PSQLException: ERROR: constraint "sym_fk_lnk_2_grp_src" of relation "sym_node_group_link" does not exist at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476) at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189) at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300) at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301) at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491) ... 10 more 2017-05-03 22:06:20,945 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists. Failed to execute: ALTER TABLE sym_conflict ADD CONSTRAINT sym_fk_cf_2_grp_lnk FOREIGN KEY (source_node_group_id, target_node_group_id) REFERENCES sym_node_group_link (source_node_group_id, target_node_group_id) 2017-10-03 22:06:20,946 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:2197569610] org.jumpmind.db.sql.SqlException: ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300) at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493) at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157) at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449) at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571) at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244) at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530) Caused by: org.postgresql.util.PSQLException: ERROR: constraint "sym_fk_cf_2_grp_lnk" for relation "sym_conflict" already exists at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476) at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189) at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300) at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301) at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491) ... 10 more OR 2017-10-03 22:05:45,665 WARN [SymmetricDS] [JdbcSqlTemplate] [symmetric-engine-startup-0] ERROR: column "data_event_action" of relation "sym_channel" already exists. Failed to execute: ALTER TABLE public.sym_channel ADD COLUMN data_event_action CHAR(1) 2017-10-03 22:05:45,666 ERROR [SymmetricDS] [AbstractSymmetricEngine] [symmetric-engine-startup-0] An error occurred while starting SymmetricDS StackTraceKey.init [SqlException:4252188107] org.jumpmind.db.sql.SqlException: ERROR: column "data_event_action" of relation "sym_channel" already exists at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:300) at org.jumpmind.db.sql.AbstractSqlTemplate.translate(AbstractSqlTemplate.java:291) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:493) at org.jumpmind.db.sql.JdbcSqlTemplate.update(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.SqlScript.execute(SqlScript.java:107) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.createOrAlterTablesIfNecessary(AbstractSymmetricDialect.java:526) at org.jumpmind.symmetric.db.AbstractSymmetricDialect.initTablesAndDatabaseObjects(AbstractSymmetricDialect.java:157) at org.jumpmind.symmetric.AbstractSymmetricEngine.setupDatabase(AbstractSymmetricEngine.java:449) at org.jumpmind.symmetric.AbstractSymmetricEngine.setup(AbstractSymmetricEngine.java:423) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:580) at org.jumpmind.symmetric.AbstractSymmetricEngine.start(AbstractSymmetricEngine.java:571) at org.jumpmind.symmetric.ClientSymmetricEngine.start(ClientSymmetricEngine.java:244) at org.jumpmind.symmetric.web.SymmetricEngineHolder$EngineStarter.run(SymmetricEngineHolder.java:530) Caused by: org.postgresql.util.PSQLException: ERROR: column "data_event_action" of relation "sym_channel" already exists at org.postgresql.core.v3.QueryExecutorImpl.receiveErrorResponse(QueryExecutorImpl.java:2476) at org.postgresql.core.v3.QueryExecutorImpl.processResults(QueryExecutorImpl.java:2189) at org.postgresql.core.v3.QueryExecutorImpl.execute(QueryExecutorImpl.java:300) at org.postgresql.jdbc.PgStatement.executeInternal(PgStatement.java:428) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:354) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:301) at org.postgresql.jdbc.PgStatement.executeCachedSql(PgStatement.java:287) at org.postgresql.jdbc.PgStatement.executeWithFlags(PgStatement.java:264) at org.postgresql.jdbc.PgStatement.execute(PgStatement.java:260) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.apache.commons.dbcp.DelegatingStatement.execute(DelegatingStatement.java:264) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:416) at org.jumpmind.db.sql.JdbcSqlTemplate$6.execute(JdbcSqlTemplate.java:401) at org.jumpmind.db.sql.JdbcSqlTemplate.execute(JdbcSqlTemplate.java:491) ... 10 more |