#define TS 32 // tile size #define WPT 8 // work per thread #define RTS (TS/WPT) // #define WPT_NR 16 // work per thread in case no remainder #define RTS_NR (TS/WPT_NR) #define PAD_X 16 #define PAD_Y 16 // single precision matrix multiplication __kernel void sgemm(__global float *A, __global float *B, __global float *C, int M, int N, int K) { // thread index const int row = get_local_id(0); // local row index of C const int col = get_local_id(1); // local column index of C const int global_row = TS * get_group_id(0) + row; // global row index of C const int global_col = TS * get_group_id(1) + col; // global column index of C __local float Asub[TS][TS]; __local float Bsub[TS][TS]; float intermediate_val[WPT]; //printf("row, col: %d, %d // global row, global col: %d, %d \n", row, col, global_row, global_col); for(int w = 0; w < WPT; w++) { intermediate_val[w] = 0.0f; } // remainder const int num_tiles = (K + TS - 1) / TS; //printf("K = %d, K % TS = %d, numtile = %d\n", K, (K % TS), num_tiles); for(int t = 0; t < num_tiles; t++){ for(int w = 0; w < WPT; w++){ const int t_row = TS * t + row; const int t_col = TS * t + col; if(global_row + w * RTS >= M || t_col >= K) Asub[row + w * RTS][col] = 0.0f; else Asub[row + w * RTS][col] = A[(global_row + w * RTS) * K + t_col]; if(t_row + w * RTS >= K || global_col >= N) Bsub[row + w * RTS][col] = 0.0f; else Bsub[row + w * RTS][col] = B[(t_row + w * RTS) * N + global_col]; } barrier(CLK_LOCAL_MEM_FENCE); for (int k = 0; k < TS; k++) { for(int w = 0; w < WPT; w++) { intermediate_val[w] += Asub[row + w * RTS][k] * Bsub[k][col]; } } barrier(CLK_LOCAL_MEM_FENCE); } for(int w = 0; w < WPT; w++) { if(global_row + w * RTS >= M || global_col >= N) continue; else C[(global_row + w * RTS) * N + global_col] = intermediate_val[w]; } } __kernel void sgemmNoRemainder(__global float *A, __global float *B, __global float *C, int M, int N, int K) { const int row = get_local_id(0); const int col = get_local_id(1); const int global_row = TS * get_group_id(0) + row; const int global_col = TS * get_group_id(1) + col; __local float Asub[TS][TS]; __local float Bsub[TS][TS]; float intermediate_val[WPT_NR]; for(int w = 0; w < WPT_NR; w++) { intermediate_val[w] = 0.0f; } // No remainder const int num_tiles = K / TS; for(int t = 0; t < num_tiles; t++){ for(int w = 0; w < WPT_NR; w++){ const int t_row = TS * t + row; const int t_col = TS * t + col; Asub[row + w * RTS_NR][col] = A[(global_row + w * RTS_NR) * K + t_col]; Bsub[row + w * RTS_NR][col] = B[(t_row + w * RTS_NR) * N + global_col]; } barrier(CLK_LOCAL_MEM_FENCE); for (int k = 0; k < TS; k++) { for(int w = 0; w < WPT_NR; w++) { intermediate_val[w] += Asub[row + w * RTS_NR][k] * Bsub[k][col]; } } barrier(CLK_LOCAL_MEM_FENCE); } for(int w = 0; w < WPT_NR; w++) { C[(global_row + w * RTS_NR) * N + global_col] = intermediate_val[w]; } } // Pad the P * Q matrix with zeroes to form a P_XL * Q_XL matrix __kernel void paddingAddZeroes(const int P, const int Q, const __global float* input, const int P_XL, const int Q_XL, __global float* output) { // thread index const int tx = get_group_id(0)*PAD_X + get_local_id(0); // 0..P_XL in blocks of PAD_X const int ty = get_group_id(1)*PAD_Y + get_local_id(1); // 0..Q_XL in blocks of PAD_Y // Check whether we are within bounds of the XL matrix if (tx < P_XL && ty < Q_XL) { // Copy the input or pad a zero float value; if (tx < P && ty < Q) { value = input[ty*P + tx]; } else { value = 0.0f; } // Store the result output[ty*P_XL + tx] = value; } } // Remove padded values from a P_XL * Q_XL matrix to form a P * Q matrix __kernel void paddingRemoveZeroes(const int P_XL, const int Q_XL, const __global float* input, const int P, const int Q, __global float* output) { // Thread identifiers const int tx = get_group_id(0)*PAD_X + get_local_id(0); // 0..P in blocks of PAD_X const int ty = get_group_id(1)*PAD_Y + get_local_id(1); // 0..Q in blocks of PAD_Y // Only store the result if within P * Q bounds if (tx < P && ty < Q) { output[ty*P + tx] = input[ty*P_XL + tx]; } }