Added reindexer for skiplist

src/Soil-Core-Tests/SoilSkipListTest.class.st

@@ -382,6 +382,39 @@ SoilSkipListTest >> testPageAddFirst [
 	self assert: page items first key equals: (#foo asSkipListKeyOfSize: 8) asInteger
 ]
 
+{ #category : #tests }
+SoilSkipListTest >> testReIndexing [
+
+	| capacity fileSizeBefore |
+	capacity := index firstPage itemCapacity.
+	1 to: capacity + 1 do: [ :n | 
+		index at: n  put: #[ 1 2 3 4 5 6 7 8 ] ].
+	index flush.
+	fileSizeBefore := index path size.
+	index reIndex.
+	self assert: index headerPage lastPageIndex equals: 2.
+	self assert: (index pageAt: 2) items size equals: 1.
+	"self assert: index path size equals: fileSizeBefore"
+]
+
+{ #category : #tests }
+SoilSkipListTest >> testReIndexingWithCleaning [
+
+	| capacity |
+	capacity := index firstPage itemCapacity.
+	1 to: capacity + 1 do: [ :n | 
+		index at: n  put: #[ 1 2 3 4 5 6 7 8 ] ].
+	index flush.
+	1 to: capacity do: [ :n | 
+		index at: n  put: SoilObjectId removed ].
+
+	index newReIndexer cleanRemoved run.
+	self assert: index headerPage lastPageIndex equals: 1.
+	self assert: (index pageAt: 1) items size equals: 1.
+	self assert: (index at: capacity + 1) asByteArray equals: #[ 1 2 3 4 5 6 7 8 ].
+
+]
+
 { #category : #tests }
 SoilSkipListTest >> testRemoveKey [
 

src/Soil-Core/SoilCopyOnWriteSkipList.class.st

@@ -17,6 +17,18 @@ SoilCopyOnWriteSkipList >> newPage [
 	^ wrapped newPage
 ]
 
+{ #category : #'as yet unclassified' }
+SoilCopyOnWriteSkipList >> reIndex [
+	wrapped reIndex. 
+	self flushCachedPages 
+]
+
+{ #category : #'as yet unclassified' }
+SoilCopyOnWriteSkipList >> reIndexUsing: itemBlock [
+	wrapped reIndexUsing: itemBlock.
+	self flushCachedPages 
+]
+
 { #category : #converting }
 SoilCopyOnWriteSkipList >> thePersistentInstance [
 	^ wrapped

src/Soil-Core/SoilIndex.class.st

@@ -42,7 +42,8 @@ SoilIndex >> basicAt: key put: anObject [
 
 { #category : #'open/close' }
 SoilIndex >> close [
-	self store close
+	self store close.
+	store := nil
 ]
 
 { #category : #enumerating }

src/Soil-Core/SoilIndexIterator.class.st

@@ -55,6 +55,13 @@ SoilIndexIterator >> atIndex: anInteger [
 	^ current value
 ]
 
+{ #category : #enumerating }
+SoilIndexIterator >> basicAssociationsDo: aBlock [
+	| item |
+	[ (item := self basicNextAssociation ) notNil ] whileTrue: [ 
+ 		aBlock value: item ] 
+]
+
 { #category : #accessing }
 SoilIndexIterator >> basicAt: key put: anObject [
 	self subclassResponsibility
@@ -293,7 +300,7 @@ SoilIndexIterator >> readVersion: anObject [
 SoilIndexIterator >> removeKey: key [ 
 	^ self 
 		at: key 
-		put: (SoilObjectId segment: 0 index: 0)
+		put: SoilObjectId removed
 ]
 
 { #category : #private }

src/Soil-Core/SoilObjectId.class.st

@@ -26,18 +26,23 @@ SoilObjectId class >> readFrom: stream [
 	segment := (stream next: 2) asInteger.
 	index := (stream next: 6) asInteger.
 	"(index = 0) ifTrue: [ SoilIndexOutOfRange signal: 'index cannot be zero' ]."
-	^ self new 
-		segment: segment;
-		index: index
+	^ self segment: segment index: index
+]
+
+{ #category : #'instance creation' }
+SoilObjectId class >> removed [
+	^ self new segment: 0; index: 0
 ]
 
 { #category : #'instance creation' }
 SoilObjectId class >> root [
-	^ RootObjectId ifNil: [ RootObjectId := self segment: 1 index: 1 ]
+	^ RootObjectId ifNil: [ RootObjectId := self new segment: 1; index: 1 ]
 ]
 
 { #category : #'instance creation' }
 SoilObjectId class >> segment: segmentId index: index [
+	(segmentId = 0 and: [ index = 0 ]) ifTrue: [ ^ self removed ].
+	(segmentId = 1 and: [ index = 1 ]) ifTrue: [ ^ self root ].
 	^ self new 
 		segment: segmentId index: index
 ]

src/Soil-Core/SoilReindexer.class.st

@@ -0,0 +1,105 @@
+Class {
+	#name : #SoilReindexer,
+	#superclass : #Object,
+	#instVars : [
+		'force',
+		'itemBlock',
+		'index',
+		'reIndexPath',
+		'newIndex',
+		'newIndexIterator'
+	],
+	#category : #'Soil-Core'
+}
+
+{ #category : #running }
+SoilReindexer >> cleanRemoved [
+	"wrap item block with a filter that removes stale values"
+	| oldBlock |
+	oldBlock := itemBlock.
+	itemBlock := [ :item |
+		item value isRemoved 
+			ifTrue: [ nil ]
+			ifFalse: [ oldBlock value: item  ] ] 
+]
+
+{ #category : #accessing }
+SoilReindexer >> force: aBoolean [ 
+	force := aBoolean
+]
+
+{ #category : #accessing }
+SoilReindexer >> index: anIndex [ 
+	index := anIndex.
+	reIndexPath := anIndex path, #reindex
+]
+
+{ #category : #initialization }
+SoilReindexer >> initialize [ 
+	super initialize.
+	force := false.
+	itemBlock := [ :item | item ]
+]
+
+{ #category : #accessing }
+SoilReindexer >> itemBlock: anObject [
+
+	itemBlock := anObject
+]
+
+{ #category : #running }
+SoilReindexer >> newIndexIterator [  
+	^ newIndexIterator ifNil: [  
+		newIndexIterator := newIndex newIterator ] 
+]
+
+{ #category : #running }
+SoilReindexer >> prepareNewIndex [ 
+	"create same index class with same parameters but a temporary path instead"
+	newIndex := index class new 
+		path: reIndexPath;
+		initializeFilesystem;
+		initializeHeaderPage;
+		maxLevel: index maxLevel;
+		keySize: index keySize;
+		valueSize: index valueSize
+]
+
+{ #category : #running }
+SoilReindexer >> replaceIndex [ 
+	"flush new index to disk and then close streams"
+	newIndex 
+		flush;
+		close.
+	"close old index so the file store gets unloaded"
+	index close.
+	"replace old index file with new index file"
+	index path delete.
+	reIndexPath moveTo: index path.
+	"reopen source index to load new index file"
+	index open.
+
+]
+
+{ #category : #running }
+SoilReindexer >> run [ 
+	self validatePath.
+	self prepareNewIndex.
+	[  
+		index newIterator basicAssociationsDo: [ :item |
+				(itemBlock value: item) ifNotNil: [ :newItem | 
+					self newIndexIterator at: newItem key put: newItem value ] ] ]
+		ifCurtailed: [ 
+			reIndexPath deleteIfAbsent: [].
+			Error signal: 'reindexing failed' ].
+	self replaceIndex 
+]
+
+{ #category : #running }
+SoilReindexer >> validatePath [
+	"forcing a reindex will remove already existing temporary path"
+	force ifTrue: [ reIndexPath deleteIfAbsent: [ ] ].
+	"if not being forced it is safer to abort if the temporary path for reindexing exists"
+	reIndexPath exists 
+		ifTrue: [ Error signal: 'path of index reindexing file already exists.' ].
+]

src/Soil-Core/SoilRemoveKeyEntry.class.st

@@ -21,7 +21,7 @@ SoilRemoveKeyEntry >> address [
 { #category : #committing }
 SoilRemoveKeyEntry >> commitIn: soil [ 
 	((soil objectRepository segmentAt: segment) indexAt: id) newIterator 
-		at: key put: (SoilObjectId segment: 0 index: 0);
+		at: key put: SoilObjectId removed;
 		updateCurrentTransaction: transactionId 
 ]
 

src/Soil-Core/SoilSkipList.class.st

@@ -95,6 +95,12 @@ SoilSkipList >> newPage [
 		yourself
 ]
 
+{ #category : #'as yet unclassified' }
+SoilSkipList >> newReIndexer [
+	^ SoilReindexer new 
+		index: self
+]
+
 { #category : #'opening/closing' }
 SoilSkipList >> open [
 	self store open
@@ -117,6 +123,18 @@ SoilSkipList >> path: aStringOrFileReference [
 	path := aStringOrFileReference asFileReference 
 ]
 
+{ #category : #'as yet unclassified' }
+SoilSkipList >> reIndex [
+	self newReIndexer run
+]
+
+{ #category : #'as yet unclassified' }
+SoilSkipList >> reIndexUsing: itemBlock [ 
+	self newReIndexer 
+		itemBlock: itemBlock;
+		run
+]
+
 { #category : #'instance creation' }
 SoilSkipList >> readPageFrom: aStream [
 	^ (SoilIndexPage readPageFrom: aStream keySize: self keySize valueSize: self valueSize)