GeezerVison

The World Wide Web, restored to 800 pixels of glory.

GeezerVison is showing github.com — GPU: implement the work-group and warp collectives for Metal · AliceO2Group/AliceO2@9ac4611

File tree

Expand file treeCollapse file tree

Open diff view settings

Filter options

GPU
- Common
  - GPUCommonAlgorithm.h
- GPUTracking
  - Base
    - GPUGeneralKernels.h
  - TPCClusterFinder

Expand file treeCollapse file tree

Open diff view settings

Collapse file

`‎GPU/Common/GPUCommonAlgorithm.h‎`

Copy file name to clipboardExpand all lines: GPU/Common/GPUCommonAlgorithm.h

+105Lines changed: 105 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -500,6 +500,111 @@ GPUdi() T warp_reduce_max(T v)`
`500`	`500`	`return __syncthreads_count(pred);`
`501`	`501`	`}`
`502`	`502`
	`503`	`+#elif defined(__METAL__)`
	`504`	`+// MSL has SIMD-group collectives only, so the work-group ones scan within each`
	`505`	`+// SIMD group and then across the group totals, which go through smem as with`
	`506`	`+// cub. The thread index is a kernel parameter in MSL, so like smem it is taken`
	`507`	`+// from the caller's scope. A 1D threadgroup is split into SIMD groups of`
	`508`	`+// consecutive threads, of GPUCA_WARP_SIZE each. The SIMD scans take 32-bit`
	`509`	`+// lanes; the narrower types are counts and are widened. Sums are returned`
	`510`	`+// promoted, as v + 0 would be, since the callers pass !!pred: a bool would`
	`511`	`+// truncate them to 0 / 1.`
	`512`	`+`
	`513`	`+template <class T>`
	`514`	`+GPUdi() auto warp_scan_inclusive_add(T v) -> decltype(+v)`
	`515`	`+{`
	`516`	`+ return simd_prefix_inclusive_sum((int32_t)v);`
	`517`	`+}`
	`518`	`+`
	`519`	`+template <class T>`
	`520`	`+GPUdi() T warp_broadcast(T v, int32_t i)`
	`521`	`+{`
	`522`	`+ return (T)simd_shuffle((int32_t)v, (ushort)i);`
	`523`	`+}`
	`524`	`+`
	`525`	`+template <class T>`
	`526`	`+GPUdi() T warp_shuffle_xor(T v, int32_t laneMask)`
	`527`	`+{`
	`528`	`+ return simd_shuffle_xor(v, (ushort)laneMask);`
	`529`	`+}`
	`530`	`+`
	`531`	`+template <int32_t Width, class T>`
	`532`	`+GPUdi() T warp_reduce_add(T v)`
	`533`	`+{`
	`534`	`+ for (int32_t m = Width / 2; m > 0; m >>= 1) {`
	`535`	`+ v += warp_shuffle_xor(v, m);`
	`536`	`+ }`
	`537`	`+ return v;`
	`538`	`+}`
	`539`	`+`
	`540`	`+template <int32_t Width, class T>`
	`541`	`+GPUdi() T warp_reduce_min(T v)`
	`542`	`+{`
	`543`	`+ for (int32_t m = Width / 2; m > 0; m >>= 1) {`
	`544`	`+ const T o = warp_shuffle_xor(v, m);`
	`545`	`+ v = o < v ? o : v;`
	`546`	`+ }`
	`547`	`+ return v;`
	`548`	`+}`
	`549`	`+`
	`550`	`+template <int32_t Width, class T>`
	`551`	`+GPUdi() T warp_reduce_max(T v)`
	`552`	`+{`
	`553`	`+ for (int32_t m = Width / 2; m > 0; m >>= 1) {`
	`554`	`+ const T o = warp_shuffle_xor(v, m);`
	`555`	`+ v = o > v ? o : v;`
	`556`	`+ }`
	`557`	`+ return v;`
	`558`	`+}`
	`559`	`+`
	`560`	`+#define work_group_scan_inclusive_add(v) work_group_scan_inclusive_add_FUNC(v, smem, iThread, nThreads)`
	`561`	`+template <class T, class S>`
	`562`	`+GPUdi() auto work_group_scan_inclusive_add_FUNC(T v, S& smem, int32_t iThread, int32_t nThreads) -> decltype(+v)`
	`563`	`+{`
	`564`	`+ const int32_t iWarp = iThread / GPUCA_WARP_SIZE;`
	`565`	`+ const int32_t nWarps = (nThreads + GPUCA_WARP_SIZE - 1) / GPUCA_WARP_SIZE;`
	`566`	`+ int32_t x = simd_prefix_inclusive_sum((int32_t)v);`
	`567`	`+ if (iThread % GPUCA_WARP_SIZE == GPUCA_WARP_SIZE - 1 \|\| iThread == nThreads - 1) {`
	`568`	`+ smem.warpPredicateSum[iWarp] = x;`
	`569`	`+ }`
	`570`	`+ GPUbarrier();`
	`571`	`+ if (iWarp == 0) {`
	`572`	`+ const int32_t s = simd_prefix_inclusive_sum(iThread < nWarps ? smem.warpPredicateSum[iThread] : 0);`
	`573`	`+ if (iThread < nWarps) {`
	`574`	`+ smem.warpPredicateSum[iThread] = s;`
	`575`	`+ }`
	`576`	`+ }`
	`577`	`+ GPUbarrier();`
	`578`	`+ if (iWarp > 0) {`
	`579`	`+ x += smem.warpPredicateSum[iWarp - 1];`
	`580`	`+ }`
	`581`	`+ GPUbarrier();`
	`582`	`+ return x;`
	`583`	`+}`
	`584`	`+`
	`585`	`+#define work_group_broadcast(v, i) work_group_broadcast_FUNC(v, i, smem, iThread)`
	`586`	`+template <class T, class S>`
	`587`	`+GPUdi() T work_group_broadcast_FUNC(T v, int32_t i, S& smem, int32_t iThread)`
	`588`	`+{`
	`589`	`+ if (iThread == i) {`
	`590`	`+ smem.tmpBroadcast = v;`
	`591`	`+ }`
	`592`	`+ GPUbarrier();`
	`593`	`+ T retVal = smem.tmpBroadcast;`
	`594`	`+ GPUbarrier();`
	`595`	`+ return retVal;`
	`596`	`+}`
	`597`	`+`
	`598`	`+#define work_group_reduce_add(v) work_group_reduce_add_FUNC(v, smem, iThread, nThreads)`
	`599`	`+template <class T, class S>`
	`600`	`+GPUdi() auto work_group_reduce_add_FUNC(T v, S& smem, int32_t iThread, int32_t nThreads) -> decltype(+v)`
	`601`	`+{`
	`602`	`+ return work_group_broadcast_FUNC(work_group_scan_inclusive_add_FUNC(v, smem, iThread, nThreads), nThreads - 1, smem, iThread);`
	`603`	`+}`
	`604`	`+`
	`605`	`+#define work_group_count(pred) ((uint32_t)work_group_reduce_add_FUNC((int32_t)(pred), smem, iThread, nThreads))`
	`606`	`+#define work_group_any(pred) (work_group_count(pred) != 0)`
	`607`	`+`
`503`	`608`	`#else`
`504`	`609`	`// Trivial implementation for the CPU`
`505`	`610`

Collapse file

`‎GPU/GPUTracking/Base/GPUGeneralKernels.h‎`

Copy file name to clipboardExpand all lines: GPU/GPUTracking/Base/GPUGeneralKernels.h

+6Lines changed: 6 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -76,6 +76,12 @@ class GPUKernelTemplate`
`76`	`76`	`int32_t tmpBroadcast;`
`77`	`77`	`int32_t warpPredicateSum[I / GPUCA_WARP_SIZE];`
`78`	`78`	`};`
	`79`	`+#elif defined(__METAL__) && defined(GPUCA_GPUCODE)`
	`80`	`+ // What the Metal work-group collectives in GPUCommonAlgorithm.h go through`
	`81`	`+ union {`
	`82`	`+ int32_t tmpBroadcast;`
	`83`	`+ int32_t warpPredicateSum[I / GPUCA_WARP_SIZE];`
	`84`	`+ };`
`79`	`85`	`#endif`
`80`	`86`	`};`
`81`	`87`

Collapse file

`‎GPU/GPUTracking/TPCClusterFinder/CfUtils.h‎`

Copy file name to clipboardExpand all lines: GPU/GPUTracking/TPCClusterFinder/CfUtils.h

+4-4Lines changed: 4 additions & 4 deletions

Original file line number	Diff line number	Diff line change
`@@ -65,7 +65,7 @@ class CfUtils`
`65`	`65`	`}`
`66`	`66`
`67`	`67`	`template <size_t BlockSize, typename SharedMemory>`
`68`		`- static GPUdi() int32_t blockPredicateScan(SharedMemory& smem, int32_t pred, int32_t* sum = nullptr)`
	`68`	`+ static GPUdi() int32_t blockPredicateScan(SharedMemory& smem, int32_t pred, int32_t* sum GPUCA_THREAD_INFO_DECL)`
`69`	`69`	`{`
`70`	`70`	`#if defined(__HIPCC__) \|\| defined(__CUDACC__)`
`71`	`71`	`int32_t iThread =`
`@@ -115,7 +115,7 @@ class CfUtils`
`115`	`115`	`}`
`116`	`116`
`117`	`117`	`template <size_t BlockSize, typename SharedMemory>`
`118`		`- static GPUdi() int32_t blockPredicateSum(SharedMemory& smem, int32_t pred)`
	`118`	`+ static GPUdi() int32_t blockPredicateSum(SharedMemory& smem, int32_t pred GPUCA_THREAD_INFO_DECL)`
`119`	`119`	`{`
`120`	`120`	`#if defined(__HIPCC__) \|\| defined(__CUDACC__)`
`121`	`121`	`int32_t iThread =`
`@@ -152,12 +152,12 @@ class CfUtils`
`152`	`152`	`}`
`153`	`153`
`154`	`154`	`template <size_t SCRATCH_PAD_WORK_GROUP_SIZE, typename SharedMemory>`
`155`		`- static GPUdi() uint16_t partition(SharedMemory& smem, uint16_t ll, bool pred, uint16_t partSize, uint16_t* newPartSize)`
	`155`	`+ static GPUdi() uint16_t partition(SharedMemory& smem, uint16_t ll, bool pred, uint16_t partSize, uint16_t* newPartSize GPUCA_THREAD_INFO_DECL)`
`156`	`156`	`{`
`157`	`157`	`bool participates = ll < partSize;`
`158`	`158`
`159`	`159`	`int32_t part;`
`160`		`- int32_t lpos = blockPredicateScan<SCRATCH_PAD_WORK_GROUP_SIZE>(smem, int32_t(!pred && participates), &part);`
	`160`	`+ int32_t lpos = blockPredicateScan<SCRATCH_PAD_WORK_GROUP_SIZE>(smem, int32_t(!pred && participates), &part GPUCA_THREAD_INFO_PROVIDE);`
`161`	`161`
`162`	`162`	`uint16_t pos = (participates && !pred) ? lpos : part;`
`163`	`163`

Collapse file

`‎GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDeconvolution.cxx‎`

Copy file name to clipboardExpand all lines: GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDeconvolution.cxx

+2-2Lines changed: 2 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -52,7 +52,7 @@ GPUdii() void GPUTPCCFDeconvolution::deconvolutionImpl(int32_t nBlocks, int32_t`
`52`	`52`
`53`	`53`	`uint16_t in3x3 = 0;`
`54`	`54`	`bool exclude3x3 = iamPeak \|\| !pos.valid();`
`55`		`- partId = CfUtils::partition<SCRATCH_PAD_WORK_GROUP_SIZE>(smem, ll, exclude3x3, SCRATCH_PAD_WORK_GROUP_SIZE, &in3x3);`
	`55`	`+ partId = CfUtils::partition<SCRATCH_PAD_WORK_GROUP_SIZE>(smem, ll, exclude3x3, SCRATCH_PAD_WORK_GROUP_SIZE, &in3x3 GPUCA_THREAD_INFO_PROVIDE);`
`56`	`56`
`57`	`57`	`if (partId < in3x3) {`
`58`	`58`	`smem.posBcast1[partId] = pos;`
`@@ -76,7 +76,7 @@ GPUdii() void GPUTPCCFDeconvolution::deconvolutionImpl(int32_t nBlocks, int32_t`
`76`	`76`	`}`
`77`	`77`
`78`	`78`	`uint16_t in5x5 = 0;`
`79`		`- partId = CfUtils::partition<SCRATCH_PAD_WORK_GROUP_SIZE>(smem, partId, peakCount > 0 && !exclude3x3, in3x3, &in5x5);`
	`79`	`+ partId = CfUtils::partition<SCRATCH_PAD_WORK_GROUP_SIZE>(smem, partId, peakCount > 0 && !exclude3x3, in3x3, &in5x5 GPUCA_THREAD_INFO_PROVIDE);`
`80`	`80`
`81`	`81`	`if (partId < in5x5) {`
`82`	`82`	`smem.posBcast1[partId] = pos;`

Collapse file

`‎GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.cxx‎`

Copy file name to clipboardExpand all lines: GPU/GPUTracking/TPCClusterFinder/GPUTPCCFPeakFinder.cxx

+1-1Lines changed: 1 addition & 1 deletion

Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,7 @@ GPUdii() bool GPUTPCCFPeakFinder::isPeak(`
`50`	`50`	`ll,`
`51`	`51`	`belowThreshold,`
`52`	`52`	`SCRATCH_PAD_WORK_GROUP_SIZE,`
`53`		`- &lookForPeaks);`
	`53`	`+ &lookForPeaks GPUCA_THREAD_INFO_PROVIDE);`
`54`	`54`
`55`	`55`	`if (partId < lookForPeaks) {`
`56`	`56`	`posBcast[partId] = pos;`

Collapse file

`‎GPU/GPUTracking/TPCClusterFinder/GPUTPCCFStreamCompaction.cxx‎`

Copy file name to clipboardExpand all lines: GPU/GPUTracking/TPCClusterFinder/GPUTPCCFStreamCompaction.cxx

+2-2Lines changed: 2 additions & 2 deletions

Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ GPUdii() void GPUTPCCFStreamCompaction::Thread<GPUTPCCFStreamCompaction::scanSta`
`36`	`36`	`pred = predicate[iThreadGlobal];`
`37`	`37`	`}`
`38`	`38`
`39`		`- int32_t nElemsInBlock = CfUtils::blockPredicateSum<GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE>(smem, pred);`
	`39`	`+ int32_t nElemsInBlock = CfUtils::blockPredicateSum<GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE>(smem, pred GPUCA_THREAD_INFO_PROVIDE);`
`40`	`40`
`41`	`41`	`int32_t lastThread = nThreads - 1;`
`42`	`42`	`if (iThread == lastThread) {`
`@@ -115,7 +115,7 @@ GPUdii() void GPUTPCCFStreamCompaction::Thread<GPUTPCCFStreamCompaction::compact`
`115`	`115`	`bool iAmDummy = (iThreadGlobal >= nElems);`
`116`	`116`
`117`	`117`	`int32_t pred = (iAmDummy) ? 0 : predicate[iThreadGlobal];`
`118`		`- int32_t offsetInBlock = CfUtils::blockPredicateScan<GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE>(smem, pred);`
	`118`	`+ int32_t offsetInBlock = CfUtils::blockPredicateScan<GPUCA_PAR_CF_SCAN_WORKGROUP_SIZE>(smem, pred, nullptr GPUCA_THREAD_INFO_PROVIDE);`
`119`	`119`
`120`	`120`	`SizeT globalOffsetOut = offsetInBlock;`
`121`	`121`	`if (iBlock > 0) {`

0 commit comments

Comments

0 (0)

You are visitor 01402520 — GeezerVison does not store the pages you look up.
Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required.