Actual source code: matmatmult.c
1: #define PETSCMAT_DLL
3: /*
4: Defines matrix-matrix product routines for pairs of SeqAIJ matrices
5: C = A * B
6: */
8: #include ../src/mat/impls/aij/seq/aij.h
9: #include ../src/mat/utils/freespace.h
10: #include petscbt.h
11: #include ../src/mat/impls/dense/seq/dense.h
15: PetscErrorCode MatMatMult_SeqAIJ_SeqAIJ(Mat A,Mat B,MatReuse scall,PetscReal fill,Mat *C)
16: {
20: if (scall == MAT_INITIAL_MATRIX){
21: MatMatMultSymbolic_SeqAIJ_SeqAIJ(A,B,fill,C);
22: }
23: MatMatMultNumeric_SeqAIJ_SeqAIJ(A,B,*C);
24: return(0);
25: }
30: PetscErrorCode MatMatMultSymbolic_SeqAIJ_SeqAIJ(Mat A,Mat B,PetscReal fill,Mat *C)
31: {
32: PetscErrorCode ierr;
33: PetscFreeSpaceList free_space=PETSC_NULL,current_space=PETSC_NULL;
34: Mat_SeqAIJ *a=(Mat_SeqAIJ*)A->data,*b=(Mat_SeqAIJ*)B->data,*c;
35: PetscInt *ai=a->i,*aj=a->j,*bi=b->i,*bj=b->j,*bjj,*ci,*cj;
36: PetscInt am=A->rmap->N,bn=B->cmap->N,bm=B->rmap->N;
37: PetscInt i,j,anzi,brow,bnzj,cnzi,nlnk,*lnk,nspacedouble=0;
38: MatScalar *ca;
39: PetscBT lnkbt;
42: /* Set up */
43: /* Allocate ci array, arrays for fill computation and */
44: /* free space for accumulating nonzero column info */
45: PetscMalloc(((am+1)+1)*sizeof(PetscInt),&ci);
46: ci[0] = 0;
47:
48: /* create and initialize a linked list */
49: nlnk = bn+1;
50: PetscLLCreate(bn,bn,nlnk,lnk,lnkbt);
52: /* Initial FreeSpace size is fill*(nnz(A)+nnz(B)) */
53: PetscFreeSpaceGet((PetscInt)(fill*(ai[am]+bi[bm])),&free_space);
54: current_space = free_space;
56: /* Determine symbolic info for each row of the product: */
57: for (i=0;i<am;i++) {
58: anzi = ai[i+1] - ai[i];
59: cnzi = 0;
60: j = anzi;
61: aj = a->j + ai[i];
62: while (j){/* assume cols are almost in increasing order, starting from its end saves computation */
63: j--;
64: brow = *(aj + j);
65: bnzj = bi[brow+1] - bi[brow];
66: bjj = bj + bi[brow];
67: /* add non-zero cols of B into the sorted linked list lnk */
68: PetscLLAdd(bnzj,bjj,bn,nlnk,lnk,lnkbt);
69: cnzi += nlnk;
70: }
72: /* If free space is not available, make more free space */
73: /* Double the amount of total space in the list */
74: if (current_space->local_remaining<cnzi) {
75: PetscFreeSpaceGet(cnzi+current_space->total_array_size,¤t_space);
76: nspacedouble++;
77: }
79: /* Copy data into free space, then initialize lnk */
80: PetscLLClean(bn,bn,cnzi,lnk,current_space->array,lnkbt);
81: current_space->array += cnzi;
82: current_space->local_used += cnzi;
83: current_space->local_remaining -= cnzi;
85: ci[i+1] = ci[i] + cnzi;
86: }
88: /* Column indices are in the list of free space */
89: /* Allocate space for cj, initialize cj, and */
90: /* destroy list of free space and other temporary array(s) */
91: PetscMalloc((ci[am]+1)*sizeof(PetscInt),&cj);
92: PetscFreeSpaceContiguous(&free_space,cj);
93: PetscLLDestroy(lnk,lnkbt);
94:
95: /* Allocate space for ca */
96: PetscMalloc((ci[am]+1)*sizeof(MatScalar),&ca);
97: PetscMemzero(ca,(ci[am]+1)*sizeof(MatScalar));
98:
99: /* put together the new symbolic matrix */
100: MatCreateSeqAIJWithArrays(((PetscObject)A)->comm,am,bn,ci,cj,ca,C);
102: /* MatCreateSeqAIJWithArrays flags matrix so PETSc doesn't free the user's arrays. */
103: /* These are PETSc arrays, so change flags so arrays can be deleted by PETSc */
104: c = (Mat_SeqAIJ *)((*C)->data);
105: c->free_a = PETSC_TRUE;
106: c->free_ij = PETSC_TRUE;
107: c->nonew = 0;
109: #if defined(PETSC_USE_INFO)
110: if (ci[am] != 0) {
111: PetscReal afill = ((PetscReal)ci[am])/(ai[am]+bi[bm]);
112: if (afill < 1.0) afill = 1.0;
113: PetscInfo3((*C),"Reallocs %D; Fill ratio: given %G needed %G.\n",nspacedouble,fill,afill);
114: PetscInfo1((*C),"Use MatMatMult(A,B,MatReuse,%G,&C) for best performance.;\n",afill);
115: } else {
116: PetscInfo((*C),"Empty matrix product\n");
117: }
118: #endif
119: return(0);
120: }
125: PetscErrorCode MatMatMultNumeric_SeqAIJ_SeqAIJ(Mat A,Mat B,Mat C)
126: {
128: PetscLogDouble flops=0.0;
129: Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data;
130: Mat_SeqAIJ *b = (Mat_SeqAIJ *)B->data;
131: Mat_SeqAIJ *c = (Mat_SeqAIJ *)C->data;
132: PetscInt *ai=a->i,*aj=a->j,*bi=b->i,*bj=b->j,*bjj,*ci=c->i,*cj=c->j;
133: PetscInt am=A->rmap->N,cm=C->rmap->N;
134: PetscInt i,j,k,anzi,bnzi,cnzi,brow,nextb;
135: MatScalar *aa=a->a,*ba=b->a,*baj,*ca=c->a;
138: /* clean old values in C */
139: PetscMemzero(ca,ci[cm]*sizeof(MatScalar));
140: /* Traverse A row-wise. */
141: /* Build the ith row in C by summing over nonzero columns in A, */
142: /* the rows of B corresponding to nonzeros of A. */
143: for (i=0;i<am;i++) {
144: anzi = ai[i+1] - ai[i];
145: for (j=0;j<anzi;j++) {
146: brow = *aj++;
147: bnzi = bi[brow+1] - bi[brow];
148: bjj = bj + bi[brow];
149: baj = ba + bi[brow];
150: nextb = 0;
151: for (k=0; nextb<bnzi; k++) {
152: if (cj[k] == bjj[nextb]){ /* ccol == bcol */
153: ca[k] += (*aa)*baj[nextb++];
154: }
155: }
156: flops += 2*bnzi;
157: aa++;
158: }
159: cnzi = ci[i+1] - ci[i];
160: ca += cnzi;
161: cj += cnzi;
162: }
163: MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);
164: MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);
166: PetscLogFlops(flops);
167: return(0);
168: }
173: PetscErrorCode MatMatMultTranspose_SeqAIJ_SeqAIJ(Mat A,Mat B,MatReuse scall,PetscReal fill,Mat *C) {
177: if (scall == MAT_INITIAL_MATRIX){
178: MatMatMultTransposeSymbolic_SeqAIJ_SeqAIJ(A,B,fill,C);
179: }
180: MatMatMultTransposeNumeric_SeqAIJ_SeqAIJ(A,B,*C);
181: return(0);
182: }
186: PetscErrorCode MatMatMultTransposeSymbolic_SeqAIJ_SeqAIJ(Mat A,Mat B,PetscReal fill,Mat *C)
187: {
189: Mat At;
190: PetscInt *ati,*atj;
193: /* create symbolic At */
194: MatGetSymbolicTranspose_SeqAIJ(A,&ati,&atj);
195: MatCreateSeqAIJWithArrays(PETSC_COMM_SELF,A->cmap->n,A->rmap->n,ati,atj,PETSC_NULL,&At);
197: /* get symbolic C=At*B */
198: MatMatMultSymbolic_SeqAIJ_SeqAIJ(At,B,fill,C);
200: /* clean up */
201: MatDestroy(At);
202: MatRestoreSymbolicTranspose_SeqAIJ(A,&ati,&atj);
203:
204: return(0);
205: }
209: PetscErrorCode MatMatMultTransposeNumeric_SeqAIJ_SeqAIJ(Mat A,Mat B,Mat C)
210: {
212: Mat_SeqAIJ *a=(Mat_SeqAIJ*)A->data,*b=(Mat_SeqAIJ*)B->data,*c=(Mat_SeqAIJ*)C->data;
213: PetscInt am=A->rmap->n,anzi,*ai=a->i,*aj=a->j,*bi=b->i,*bj,bnzi,nextb;
214: PetscInt cm=C->rmap->n,*ci=c->i,*cj=c->j,crow,*cjj,i,j,k;
215: PetscLogDouble flops=0.0;
216: MatScalar *aa=a->a,*ba,*ca=c->a,*caj;
217:
219: /* clear old values in C */
220: PetscMemzero(ca,ci[cm]*sizeof(MatScalar));
222: /* compute A^T*B using outer product (A^T)[:,i]*B[i,:] */
223: for (i=0;i<am;i++) {
224: bj = b->j + bi[i];
225: ba = b->a + bi[i];
226: bnzi = bi[i+1] - bi[i];
227: anzi = ai[i+1] - ai[i];
228: for (j=0; j<anzi; j++) {
229: nextb = 0;
230: crow = *aj++;
231: cjj = cj + ci[crow];
232: caj = ca + ci[crow];
233: /* perform sparse axpy operation. Note cjj includes bj. */
234: for (k=0; nextb<bnzi; k++) {
235: if (cjj[k] == *(bj+nextb)) { /* ccol == bcol */
236: caj[k] += (*aa)*(*(ba+nextb));
237: nextb++;
238: }
239: }
240: flops += 2*bnzi;
241: aa++;
242: }
243: }
245: /* Assemble the final matrix and clean up */
246: MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);
247: MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);
248: PetscLogFlops(flops);
249: return(0);
250: }
254: PetscErrorCode MatMatMult_SeqAIJ_SeqDense(Mat A,Mat B,MatReuse scall,PetscReal fill,Mat *C)
255: {
259: if (scall == MAT_INITIAL_MATRIX){
260: MatMatMultSymbolic_SeqAIJ_SeqDense(A,B,fill,C);
261: }
262: MatMatMultNumeric_SeqAIJ_SeqDense(A,B,*C);
263: return(0);
264: }
268: PetscErrorCode MatMatMultSymbolic_SeqAIJ_SeqDense(Mat A,Mat B,PetscReal fill,Mat *C)
269: {
273: MatMatMultSymbolic_SeqDense_SeqDense(A,B,0.0,C);
274: return(0);
275: }
279: PetscErrorCode MatMatMultNumeric_SeqAIJ_SeqDense(Mat A,Mat B,Mat C)
280: {
281: Mat_SeqAIJ *a=(Mat_SeqAIJ*)A->data;
283: PetscScalar *b,*c,r1,r2,r3,r4,*b1,*b2,*b3,*b4;
284: MatScalar *aa;
285: PetscInt cm=C->rmap->n, cn=B->cmap->n, bm=B->rmap->n, col, i,j,n,*aj, am = A->rmap->n;
286: PetscInt am2 = 2*am, am3 = 3*am, bm4 = 4*bm,colam;
289: if (!cm || !cn) return(0);
290: if (bm != A->cmap->n) SETERRQ2(PETSC_ERR_ARG_SIZ,"Number columns in A %D not equal rows in B %D\n",A->cmap->n,bm);
291: if (A->rmap->n != C->rmap->n) SETERRQ2(PETSC_ERR_ARG_SIZ,"Number rows in C %D not equal rows in A %D\n",C->rmap->n,A->rmap->n);
292: if (B->cmap->n != C->cmap->n) SETERRQ2(PETSC_ERR_ARG_SIZ,"Number columns in B %D not equal columns in C %D\n",B->cmap->n,C->cmap->n);
293: MatGetArray(B,&b);
294: MatGetArray(C,&c);
295: b1 = b; b2 = b1 + bm; b3 = b2 + bm; b4 = b3 + bm;
296: for (col=0; col<cn-4; col += 4){ /* over columns of C */
297: colam = col*am;
298: for (i=0; i<am; i++) { /* over rows of C in those columns */
299: r1 = r2 = r3 = r4 = 0.0;
300: n = a->i[i+1] - a->i[i];
301: aj = a->j + a->i[i];
302: aa = a->a + a->i[i];
303: for (j=0; j<n; j++) {
304: r1 += (*aa)*b1[*aj];
305: r2 += (*aa)*b2[*aj];
306: r3 += (*aa)*b3[*aj];
307: r4 += (*aa++)*b4[*aj++];
308: }
309: c[colam + i] = r1;
310: c[colam + am + i] = r2;
311: c[colam + am2 + i] = r3;
312: c[colam + am3 + i] = r4;
313: }
314: b1 += bm4;
315: b2 += bm4;
316: b3 += bm4;
317: b4 += bm4;
318: }
319: for (;col<cn; col++){ /* over extra columns of C */
320: for (i=0; i<am; i++) { /* over rows of C in those columns */
321: r1 = 0.0;
322: n = a->i[i+1] - a->i[i];
323: aj = a->j + a->i[i];
324: aa = a->a + a->i[i];
326: for (j=0; j<n; j++) {
327: r1 += (*aa++)*b1[*aj++];
328: }
329: c[col*am + i] = r1;
330: }
331: b1 += bm;
332: }
333: PetscLogFlops(cn*(2*a->nz));
334: MatRestoreArray(B,&b);
335: MatRestoreArray(C,&c);
336: MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);
337: MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);
338: return(0);
339: }
341: /*
342: Note very similar to MatMult_SeqAIJ(), should generate both codes from same base
343: */
346: PetscErrorCode MatMatMultNumericAdd_SeqAIJ_SeqDense(Mat A,Mat B,Mat C)
347: {
348: Mat_SeqAIJ *a=(Mat_SeqAIJ*)A->data;
350: PetscScalar *b,*c,r1,r2,r3,r4,*b1,*b2,*b3,*b4;
351: MatScalar *aa;
352: PetscInt cm=C->rmap->n, cn=B->cmap->n, bm=B->rmap->n, col, i,j,n,*aj, am = A->rmap->n,*ii,arm;
353: PetscInt am2 = 2*am, am3 = 3*am, bm4 = 4*bm,colam,*ridx;
356: if (!cm || !cn) return(0);
357: MatGetArray(B,&b);
358: MatGetArray(C,&c);
359: b1 = b; b2 = b1 + bm; b3 = b2 + bm; b4 = b3 + bm;
361: if (a->compressedrow.use){ /* use compressed row format */
362: for (col=0; col<cn-4; col += 4){ /* over columns of C */
363: colam = col*am;
364: arm = a->compressedrow.nrows;
365: ii = a->compressedrow.i;
366: ridx = a->compressedrow.rindex;
367: for (i=0; i<arm; i++) { /* over rows of C in those columns */
368: r1 = r2 = r3 = r4 = 0.0;
369: n = ii[i+1] - ii[i];
370: aj = a->j + ii[i];
371: aa = a->a + ii[i];
372: for (j=0; j<n; j++) {
373: r1 += (*aa)*b1[*aj];
374: r2 += (*aa)*b2[*aj];
375: r3 += (*aa)*b3[*aj];
376: r4 += (*aa++)*b4[*aj++];
377: }
378: c[colam + ridx[i]] += r1;
379: c[colam + am + ridx[i]] += r2;
380: c[colam + am2 + ridx[i]] += r3;
381: c[colam + am3 + ridx[i]] += r4;
382: }
383: b1 += bm4;
384: b2 += bm4;
385: b3 += bm4;
386: b4 += bm4;
387: }
388: for (;col<cn; col++){ /* over extra columns of C */
389: colam = col*am;
390: arm = a->compressedrow.nrows;
391: ii = a->compressedrow.i;
392: ridx = a->compressedrow.rindex;
393: for (i=0; i<arm; i++) { /* over rows of C in those columns */
394: r1 = 0.0;
395: n = ii[i+1] - ii[i];
396: aj = a->j + ii[i];
397: aa = a->a + ii[i];
399: for (j=0; j<n; j++) {
400: r1 += (*aa++)*b1[*aj++];
401: }
402: c[col*am + ridx[i]] += r1;
403: }
404: b1 += bm;
405: }
406: } else {
407: for (col=0; col<cn-4; col += 4){ /* over columns of C */
408: colam = col*am;
409: for (i=0; i<am; i++) { /* over rows of C in those columns */
410: r1 = r2 = r3 = r4 = 0.0;
411: n = a->i[i+1] - a->i[i];
412: aj = a->j + a->i[i];
413: aa = a->a + a->i[i];
414: for (j=0; j<n; j++) {
415: r1 += (*aa)*b1[*aj];
416: r2 += (*aa)*b2[*aj];
417: r3 += (*aa)*b3[*aj];
418: r4 += (*aa++)*b4[*aj++];
419: }
420: c[colam + i] += r1;
421: c[colam + am + i] += r2;
422: c[colam + am2 + i] += r3;
423: c[colam + am3 + i] += r4;
424: }
425: b1 += bm4;
426: b2 += bm4;
427: b3 += bm4;
428: b4 += bm4;
429: }
430: for (;col<cn; col++){ /* over extra columns of C */
431: for (i=0; i<am; i++) { /* over rows of C in those columns */
432: r1 = 0.0;
433: n = a->i[i+1] - a->i[i];
434: aj = a->j + a->i[i];
435: aa = a->a + a->i[i];
437: for (j=0; j<n; j++) {
438: r1 += (*aa++)*b1[*aj++];
439: }
440: c[col*am + i] += r1;
441: }
442: b1 += bm;
443: }
444: }
445: PetscLogFlops(cn*2*a->nz);
446: MatRestoreArray(B,&b);
447: MatRestoreArray(C,&c);
448: return(0);
449: }