Skip to content

Commit

Permalink
CORE-189: streamline ENTITY_ATTRIBUTE_TEMP temp table (#3119)
Browse files Browse the repository at this point in the history
  • Loading branch information
@davidangb
davidangb authored Nov 21, 2024
1 parent d982f1f commit 0c7a057
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 42 deletions.
2 changes: 2 additions & 0 deletions core/src/main/resources/org/broadinstitute/dsde/rawls/liquibase/changelog.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,4 +126,6 @@
<include file="changesets/20240723_workspace_settings.xml" relativeToChangelogFile="true"/>
<include file="changesets/20240820_submission_cost_cap_threshold.xml" relativeToChangelogFile="true"/>
<include file="changesets/20240830_workflow_cost.xml" relativeToChangelogFile="true"/>
<include file="changesets/20241106_streamline_entity_attr_temp_table_procedure.xml" relativeToChangelogFile="true"/>
<include file="changesets/20241106_streamline_workspace_attr_temp_table_procedure.xml" relativeToChangelogFile="true"/>
</databaseChangeLog>
31 changes: 31 additions & 0 deletions .../dsde/rawls/liquibase/changesets/20241106_streamline_entity_attr_temp_table_procedure.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<databaseChangeLog logicalFilePath="dummy"
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd">

<changeSet logicalFilePath="dummy" author="davidan" id="streamline_entity_attr_temp_table">
<createProcedure replaceIfExists="true" procedureName="createEntityAttributeTempTable">
CREATE PROCEDURE createEntityAttributeTempTable ()
BEGIN
create temporary table ENTITY_ATTRIBUTE_TEMP (
id bigint unsigned NOT NULL AUTO_INCREMENT primary key,
namespace varchar(32) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,
name varchar(200) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,
value_string text,
value_json longtext,
value_number double DEFAULT NULL,
value_boolean bit(1) DEFAULT NULL,
value_entity_ref bigint unsigned DEFAULT NULL,
list_index int DEFAULT NULL,
list_length int DEFAULT NULL,
owner_id bigint unsigned NOT NULL,
deleted bit(1) DEFAULT false,
deleted_date timestamp NULL DEFAULT NULL
);
END
</createProcedure>
</changeSet>

</databaseChangeLog>
32 changes: 32 additions & 0 deletions ...de/rawls/liquibase/changesets/20241106_streamline_workspace_attr_temp_table_procedure.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<databaseChangeLog logicalFilePath="dummy"
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:ext="http://www.liquibase.org/xml/ns/dbchangelog-ext"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.4.xsd">

<changeSet logicalFilePath="dummy" author="davidan" id="streamline_workspace_attr_temp_table">
<createProcedure replaceIfExists="true" procedureName="createWorkspaceAttributeTempTable">
CREATE PROCEDURE createWorkspaceAttributeTempTable ()
BEGIN
create temporary table WORKSPACE_ATTRIBUTE_TEMP (
id bigint unsigned NOT NULL AUTO_INCREMENT primary key,
namespace varchar(32) NOT NULL,
name varchar(200) NOT NULL,
value_string text,
value_json longtext,
value_number double DEFAULT NULL,
value_boolean bit(1) DEFAULT NULL,
value_entity_ref bigint unsigned DEFAULT NULL,
list_index int DEFAULT NULL,
list_length int DEFAULT NULL,
owner_id BINARY(16) NOT NULL,
deleted bit(1) DEFAULT false,
deleted_date timestamp NULL DEFAULT NULL
);
END
</createProcedure>
</changeSet>


</databaseChangeLog>
59 changes: 17 additions & 42 deletions core/src/main/scala/org/broadinstitute/dsde/rawls/dataaccess/slick/AttributeComponent.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,7 @@ trait AttributeRecord[OWNER_ID] {
val deletedDate: Option[Timestamp]
}

trait AttributeScratchRecord[OWNER_ID] extends AttributeRecord[OWNER_ID] {
val transactionId: String
}
trait AttributeScratchRecord[OWNER_ID] extends AttributeRecord[OWNER_ID]

case class EntityAttributeRecord(id: Long,
ownerId: Long, // entity id
Expand Down Expand Up @@ -64,8 +62,7 @@ case class EntityAttributeTempRecord(id: Long,
listIndex: Option[Int],
listLength: Option[Int],
deleted: Boolean,
deletedDate: Option[Timestamp],
transactionId: String
deletedDate: Option[Timestamp]
) extends AttributeScratchRecord[Long]

case class WorkspaceAttributeRecord(id: Long,
Expand Down Expand Up @@ -95,8 +92,7 @@ case class WorkspaceAttributeTempRecord(id: Long,
listIndex: Option[Int],
listLength: Option[Int],
deleted: Boolean,
deletedDate: Option[Timestamp],
transactionId: String
deletedDate: Option[Timestamp]
) extends AttributeScratchRecord[UUID]

case class SubmissionAttributeRecord(id: Long,
Expand Down Expand Up @@ -148,9 +144,7 @@ trait AttributeComponent {
abstract class AttributeScratchTable[OWNER_ID: TypedType, RECORD <: AttributeScratchRecord[OWNER_ID]](
tag: Tag,
tableName: String
) extends AttributeTable[OWNER_ID, RECORD](tag, tableName) {
def transactionId = column[String]("transaction_id")
}
) extends AttributeTable[OWNER_ID, RECORD](tag, tableName)

class EntityAttributeTable(shard: ShardId)(tag: Tag)
extends AttributeTable[Long, EntityAttributeRecord](tag, s"ENTITY_ATTRIBUTE_$shard") {
Expand Down Expand Up @@ -236,8 +230,7 @@ trait AttributeComponent {
listIndex,
listLength,
deleted,
deletedDate,
transactionId
deletedDate
) <> (EntityAttributeTempRecord.tupled, EntityAttributeTempRecord.unapply)
}

Expand All @@ -255,8 +248,7 @@ trait AttributeComponent {
listIndex,
listLength,
deleted,
deletedDate,
transactionId
deletedDate
) <> (WorkspaceAttributeTempRecord.tupled, WorkspaceAttributeTempRecord.unapply)
}

Expand Down Expand Up @@ -315,14 +307,13 @@ trait AttributeComponent {
Option[Int],
Option[Int],
Boolean,
Option[Timestamp],
String
Option[Timestamp]
) => TEMP_RECORD
) extends TableQuery[T](cons) {
def insertScratchAttributes(attributeRecs: Seq[RECORD])(transactionId: String): WriteAction[Int] =
batchInsertAttributes(attributeRecs, transactionId)
def insertScratchAttributes(attributeRecs: Seq[RECORD])(): WriteAction[Int] =
batchInsertAttributes(attributeRecs)

def batchInsertAttributes(attributes: Seq[RECORD], transactionId: String) =
def batchInsertAttributes(attributes: Seq[RECORD]) =
insertInBatches(
this,
attributes.map(rec =>
Expand All @@ -339,8 +330,7 @@ trait AttributeComponent {
rec.listIndex,
rec.listLength,
rec.deleted,
rec.deletedDate,
transactionId
rec.deletedDate
)
)
)
Expand Down Expand Up @@ -597,7 +587,7 @@ trait AttributeComponent {
def patchAttributesAction(inserts: Traversable[RECORD],
updates: Traversable[RECORD],
deleteIds: Traversable[Long],
insertFunction: Seq[RECORD] => String => WriteAction[Int],
insertFunction: Seq[RECORD] => () => WriteAction[Int],
tracingContext: RawlsTracingContext
) =
traceDBIOWithParent("patchAttributesAction", tracingContext) { span =>
Expand Down Expand Up @@ -705,7 +695,7 @@ trait AttributeComponent {
*/
def rewriteAttrsAction(attributesToSave: Traversable[RECORD],
existingAttributes: Traversable[RECORD],
insertFunction: Seq[RECORD] => String => WriteAction[Int]
insertFunction: Seq[RECORD] => () => WriteAction[Int]
): ReadWriteAction[Set[OWNER_ID]] =
traceDBIOWithParent("AttributeComponent.rewriteAttrsAction", RawlsTracingContext(Option.empty)) {
tracingContext =>
Expand All @@ -714,8 +704,6 @@ trait AttributeComponent {

// note that currently-existing attributes will have a populated id e.g. "1234", but to-save will have an id of "0"
// therefore, we use this ComparableRecord class which omits the id when checking equality between existing and to-save.
// note this does not include transactionId for AttributeScratchRecords. We do not expect AttributeScratchRecords
// here, and transactionId will eventually be going away, so don't bother
object ComparableRecord {
def fromRecord(rec: RECORD): ComparableRecord =
new ComparableRecord(
Expand Down Expand Up @@ -795,15 +783,14 @@ trait AttributeComponent {
// attributes.

// updateInMasterAction: updates any row in *_ATTRIBUTE that also exists in *_ATTRIBUTE_SCRATCH
def updateInMasterAction(transactionId: String) = {
def updateInMasterAction() = {
val joinTableName = getTempOrScratchTableName(baseTableRow.tableName)

sql"""
update #${baseTableRow.tableName} a
join #${joinTableName} ta
on (a.namespace, a.name, a.owner_id, ifnull(a.list_index, 0)) =
(ta.namespace, ta.name, ta.owner_id, ifnull(ta.list_index, 0))
and ta.transaction_id = $transactionId
set a.value_string=ta.value_string,
a.value_number=ta.value_number,
a.value_boolean=ta.value_boolean,
Expand All @@ -815,22 +802,10 @@ trait AttributeComponent {
""".as[Int]
}

def clearAttributeScratchTableAction(transactionId: String) = {
val joinTableName = getTempOrScratchTableName(baseTableRow.tableName)

if (joinTableName.endsWith("SCRATCH"))
sqlu"""delete from #${joinTableName} where transaction_id = $transactionId"""
else DBIO.successful(0)
}

def updateAction(insertIntoScratchFunction: String => WriteAction[Int], tracingContext: RawlsTracingContext) =
def updateAction(insertIntoScratchFunction: () => WriteAction[Int], tracingContext: RawlsTracingContext) =
traceDBIOWithParent("updateAction", tracingContext) { span =>
val transactionId = UUID.randomUUID().toString
traceDBIOWithParent("insertIntoScratchFunction", span)(_ => insertIntoScratchFunction(transactionId)) andThen
traceDBIOWithParent("updateInMasterAction", span)(_ => updateInMasterAction(transactionId)) andThen
traceDBIOWithParent("clearAttributeScratchTableAction", span)(_ =>
clearAttributeScratchTableAction(transactionId)
)
traceDBIOWithParent("insertIntoScratchFunction", span)(_ => insertIntoScratchFunction()) andThen
traceDBIOWithParent("updateInMasterAction", span)(_ => updateInMasterAction())
}
}

Expand Down

0 comments on commit 0c7a057

Please sign in to comment.