/* @(#)comm.c 1.1 94/10/31 SMI */ /* Copyright (c) 1984 AT&T */ /* All Rights Reserved */ /* THIS IS UNPUBLISHED PROPRIETARY SOURCE CODE OF AT&T */ /* The copyright notice above does not evidence any */ /* actual or intended publication of such source code. */ /* #ident "@(#)kern-port:nudnix/comm.c 10.31" */ /* These are the communications routines called from the UNIX kernel. * They deal with send and receive descriptors, and with sending * and receiving messages. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Don't let RFS start unless we have enough resources: * SNDD: one for rfdaemon, one for mount, one for request, one for cache * RCVD: mount, signals, rfdaemon, cache, one to do something * MINGDP: at least one circuit */ #define MINSNDD 4 #define MINRCVD 5 #define MINGDP 1 /* Global variables used by this routine. */ int rdfree; /* number of free receive descriptors */ int sdfree; /* number of free send descriptors */ sndd_t sdfreelist; /* free send desc link list */ rcvd_t rdfreelist; /* free recv desc link list */ rcvd_t sigrd; /* rd for signals */ rcvd_t cfrd; /* rd for mounts */ static ushort connid; int sdlowmark; /* sd free count low mark */ int serverslp = 0; /* server sleep for buffer flag */ int clientslp = 0; /* client sleep for buffer flag */ mblk_t *server_bp = NULL; /* server stream buffer */ mblk_t *client_bp = NULL; /* client stream buffer */ int nserverbuf = 1; /* number of 2k server stream buffer */ sndd_t cache_sd; /* SD for sending cache disable messages */ rcvd_t cache_rd; /* RD for receiving cache disable responses */ extern rcvd_t rd_recover; /* recovery receive descriptor */ extern void strunbcall(); extern void del_rduser(); void commdinit(); /* * create a send descriptor. */ sndd_t cr_sndd () { register sndd_t retsndd; register int s; extern int nservers; s = splrf(); /* Make sure every server can get a send-descriptor. */ if ((retsndd = sdfreelist) == (sndd_t) NULL) { (void) splx (s); return ((sndd_t) NULL); } /* Got a free sndd. */ DUPRINT2(DB_SD_RD, "cr_sndd: sd %x\n",retsndd); sdfreelist = retsndd->sd_next; sdfree--; retsndd->sd_refcnt = 1; retsndd->sd_queue = NULL; retsndd->sd_copycnt = 0; retsndd->sd_connid = 0; retsndd->sd_sindex = 0; retsndd->sd_stat = SDUSED; retsndd->sd_fhandle = 0; retsndd->sd_count = 0; retsndd->sd_offset = 0; (void) splx(s); return (retsndd); } set_sndd (sd, queue, index, nconnid) register sndd_t sd; int nconnid; queue_t *queue; index_t index; { register int s; DUPRINT5(DB_SD_RD, "set_sndd: sndd %x queue %x index %x proc %x\n", sd, queue, index,u.u_procp); ASSERT(sd->sd_stat != SDUNUSED); s = splrf(); sd->sd_queue = queue; sd->sd_sindex = index; sd->sd_connid = nconnid; (void) splx(s); } free_sndd (sd) sndd_t sd; { int s; DUPRINT3(DB_SD_RD, "free_sndd: sd %x, proc %x \n",sd,u.u_procp); ASSERT(sd->sd_stat != SDUNUSED); s = splrf(); sd->sd_next = sdfreelist; sdfreelist = sd; sd->sd_stat = SDUNUSED; sd->sd_connid = 0; sdfree++; (void) splx(s); } int del_sndd (sd) sndd_t sd; { int error = 0; DUPRINT2(DB_SD_RD, "del_sndd: sd %x\n", sd); ASSERT(sd->sd_stat != SDUNUSED); if(sd->sd_refcnt > 1) { sd->sd_refcnt--; return(0); } error = du_iput(sd, u.u_cred); if (error == ENOLINK) error = 0; if (!error) free_sndd(sd); DUPRINT3(DB_SYSCALL, "del_sndd: DUIPUT sd %x, result %d\n", sd, error); return (error); } /* send a message. * set up all the proper fields in the message. Fill in the gift * field in the message if there is a gift. */ int sndmsg (sd, bp, bytes, gift) sndd_t sd; /* which send descriptor to send on */ mblk_t *bp; /* message pointer */ int bytes; /* how much of the buffer to send */ rcvd_t gift; /* gift, if there is one */ { register struct message *msg; register struct gdp *tgdp; queue_t *rq, *wq; register int s; DUPRINT4(DB_COMM,"sndmsg: sd %x bp %x gift %x\n", sd, bp, gift); DUPRINT4(DB_COMM, "BEFORE sndmsg: m_dest %x m_connid %x m_gindex %x\n", sd->sd_sindex,sd->sd_connid,rdtoinx (gift)); ASSERT(sd && sd->sd_stat != SDUNUSED); if (sd->sd_stat & SDLINKDOWN) { DUPRINT1(DB_RECOVER, "sndmsg: trying to send over dead link\n"); freemsg (bp); return (ENOLINK); } rq = sd->sd_queue; ASSERT(rq); tgdp = (struct gdp *)rq->q_ptr; wq = WR(rq); s = splrf(); while (!canput(wq)) { DUPRINT2(DB_GDPERR, "sndmsg: queue full on sd %x\n", sd); (void) sleep((caddr_t) tgdp, PZERO-1); if (sd->sd_stat & SDLINKDOWN) { DUPRINT1(DB_RECOVER, "sndmsg: trying to send over dead link\n"); freemsg (bp); (void) splx(s); return (ENOLINK); } } (void) splx(s); msg = (struct message *) bp->b_rptr; ((struct request *) PTOMSG(bp->b_rptr))->rq_pid = u.u_procp->p_pid; /* store the pid for signals */ bp->b_wptr = bp->b_rptr + sizeof(struct message) + bytes; msg->m_dest = sd->sd_sindex; msg->m_connid = sd->sd_connid; msg->m_cmd = 0; /* not used */ msg->m_stat |= VER1; if (gift) { msg->m_stat |= GIFT; msg->m_gindex = rdtoinx (gift); msg->m_gconnid = gift->rd_connid; ASSERT(gift->rd_stat != RDUNUSED); /* Keep track of who gets gift. */ if (gift->rd_qtype & SPECIFIC) (struct queue *) gift->rd_user_list = rq; /* (Keep track of GENERAL RDs in make_gift.) */ } msg->m_size = bytes + sizeof (struct message); DUPRINT4(DB_COMM, "sndmsg: m_dest %x m_connid %x m_gindex %x\n", msg->m_dest,msg->m_connid,msg->m_gindex); /* * convert RFS headers and data to canonical forms */ rftocanon(bp, tgdp->hetero); putq(wq, bp); return (0); } /* * Get a buffer. Doesn't return until it has the buffer. Saves signals * which may occur while sleeping waiting for a buffer. */ mblk_t * rfs_getbuf(size, bpri) int size; int bpri; /* buffer allocation priority */ { int sig; char cursig; mblk_t *bp; SAVE_SIG(u.u_procp, sig, cursig); while ((bp = alocbuf(size, bpri)) == NULL) ACCUM_SIG(u.u_procp, sig, cursig); RESTORE_SIG(u.u_procp, sig, cursig); return(bp); } mblk_t * alocbuf (size, bpri) int size; int bpri; /* buffer allocation priority */ { register mblk_t *mbp; extern setrun(); ASSERT (size <= MSGBUFSIZE); while ((mbp = allocb (size + sizeof (struct message), bpri)) == NULL) { DUPRINT1(DB_GDPERR, "alocbuf: allocb fail, using bufcall()\n"); /* wait for buffer to become available */ if (bufcall((uint)(size + sizeof(struct message)), bpri, setrun, (long) u.u_procp) == 0) { DUPRINT1(DB_GDPERR, "alocbuf: bufcall fail\n"); return(NULL); } if (sleep((caddr_t)&(u.u_procp->p_flag), PREMOTE|PCATCH) != 0) { /* wake up due to signal */ strunbcall(size + sizeof(struct message), u.u_procp); DUPRINT1(DB_GDPERR, "alocbuf: wake up by signal\n"); return(NULL); } strunbcall(size + sizeof(struct message), u.u_procp); } mbp->b_wptr += sizeof(struct message); bzero((char *)mbp->b_rptr, sizeof(struct message)+ sizeof(struct response)-DATASIZE); return (mbp); } /* * allocate a send buffer -- server side, uses private buffer pool if * none available in general pool. Note: doesn't return until it has * a buffer. */ mblk_t * salocbuf (size, bpri) int size; int bpri; /* buffer allocation priority */ { register mblk_t *mbp; register mblk_t *nbp; int s; extern mblk_t *server_bp; extern wake_serverbp(); ASSERT (size <= MSGBUFSIZE); while ((mbp = allocb (size + sizeof (struct message), bpri)) == NULL) { /* server process will use its own stream buffer */ for (mbp = server_bp; mbp; mbp = mbp->b_next) { if (mbp->b_datap->db_ref == 1) { DUPRINT2(DB_GDPERR, "alocbuf: use serv buf %x\n", mbp); if ((nbp = dupmsg(mbp)) == NULL) break; nbp->b_rptr = nbp->b_datap->db_base; nbp->b_wptr = nbp->b_datap->db_base + sizeof(struct message); bzero((char *)nbp->b_rptr, sizeof(struct message) + sizeof(struct response)-DATASIZE); return (nbp); } } DUPRINT1(DB_GDPERR, "alocbuf: fail to use server buffer\n"); s = splrf(); if (serverslp == 0) { serverslp++; /* sleep for one sec */ timeout((int (*)()) wake_serverbp, (caddr_t)0, 100); } (void) splx(s); (void) sleep((caddr_t)&server_bp, PZERO); } mbp->b_wptr += sizeof(struct message); ((struct message *)mbp->b_rptr)->m_stat = 0; return (mbp); } wake_serverbp() { serverslp = 0; wakeup((caddr_t)&server_bp); } wake_clientbp() { clientslp = 0; wakeup((caddr_t)&client_bp); } /* * Reuse incoming message buffer for response if the size * is large enough for the response. * Otherwise, allocate a buffer big enough for the response. * This will prevent possible deadlock when server fails to get a * buffer for sending response. */ mblk_t * reusebuf (bp, size) mblk_t *bp; int size; { mblk_t *nbp; if (size <= (bp->b_datap->db_lim - bp->b_datap->db_base) && bp->b_datap->db_ref == 1) { /* reuse the incoming buffer */ nbp = dupb(bp); if (nbp != NULL) { nbp->b_rptr = nbp->b_datap->db_base; nbp->b_wptr = nbp->b_datap->db_base + sizeof(struct message); ((struct message *)nbp->b_rptr)->m_stat = 0; return (nbp); } } /* fail to reuse the buffer */ /* allocate a buffer that is big enough */ nbp = rfs_getbuf(size, BPRI_MED); return (nbp); } /* * getcbp = get client buffer pointer. Called by copyin to * ensure that a buffer will be returned and thus prevent deadlock */ mblk_t * getcbp(size) int size; { int s; mblk_t *nbp = NULL; extern wake_clientbp(); /* allocate a buffer that is big enough * if you fail to allocate a buffer try * to use the preallocated client_bp * If you fail go allocate client_bp sleep for .5 sec * and try reusing your original buffer. * this is an end condition and should rarely happen * depending on the number of buffers configured. */ while ((nbp = allocb (size + sizeof (struct message), BPRI_MED)) == NULL) { if (client_bp->b_datap->db_ref == 1) { if ((nbp = dupmsg(client_bp)) == NULL) continue; nbp->b_rptr = nbp->b_datap->db_base; nbp->b_wptr = nbp->b_datap->db_base + sizeof(struct message); bzero((char *)nbp->b_rptr, sizeof(struct message)+ sizeof(struct response)-DATASIZE); return (nbp); } s = splrf(); if (clientslp == 0) { clientslp++; timeout((int (*)()) wake_clientbp, (caddr_t)0, 60); } (void) splx(s); (void) sleep ((caddr_t)&client_bp, PZERO); continue; } bzero((char *)nbp->b_rptr, sizeof(struct message)+ sizeof(struct response)-DATASIZE); return (nbp); } /* * Create a receive descriptor. */ rcvd_t cr_rcvd (qsize, type) int qsize; /* max no. messages for this rcvd */ char type; { register rcvd_t retrcvd; /* return value of the function */ if ((retrcvd = rdfreelist) != NULL) { rdfreelist = retrcvd->rd_next; rdfree--; retrcvd->rd_qsize = qsize; retrcvd->rd_qcnt = 0; retrcvd->rd_vnode = NULL; retrcvd->rd_refcnt = 1; retrcvd->rd_act_cnt = 0; retrcvd->rd_qtype = type; retrcvd->rd_user_list = NULL; retrcvd->rd_connid = connid++; retrcvd->rd_rcvdq.qc_head = NULL; retrcvd->rd_stat = RDUSED; initsema(&retrcvd->rd_qslp, 0); DUPRINT3(DB_SD_RD,"cr_rcvd: rcvd %x index %x\n", retrcvd-rcvd, retrcvd); } else DUPRINT1(DB_SD_RD, "cr_rcvd: out of rcvd\n"); return (retrcvd); } /* * Decrement the count on a receive descriptor. * If count goes to zero, free it and clean text. * If sp is non-null, the rd_user structs associated with * sp->s_mntindx will be released. */ del_rcvd (rd, sp) register rcvd_t rd; struct serv_proc *sp; { DUPRINT4(DB_SD_RD, "del_rcvd: rd %x count %x connid %x\n", rd, rd->rd_refcnt,rd->rd_connid); ASSERT(rd); ASSERT(rd->rd_stat != RDUNUSED); if (sp) del_rduser(rd, sp); if (--rd->rd_refcnt) return; free_rcvd(rd); return; } /* * Return a receive descriptor to the freelist. */ free_rcvd (rd) register rcvd_t rd; { DUPRINT2(DB_SD_RD, "free_rcvd: rd %x\n", rd); ASSERT (rd && rd->rd_stat != RDUNUSED); rd->rd_stat = RDUNUSED; rdfree++; ASSERT(!((rd->rd_qtype & GENERAL) && (rd->rd_user_list != NULL))); rd->rd_user_list = NULL; rd->rd_next = rdfreelist; rdfreelist = rd; } /* * dequeue a message off of the receive queue. * Construct a gift from the message if there is one. * The error recovery for no more send descriptors may cause * this routine to sleep waiting for one. * Returns RCVQEMP or SUCCESS. */ dequeue (rd, bufp, sd, size) register rcvd_t rd; mblk_t **bufp; /* output - the address of the message buffer */ sndd_t sd; /* output - the gift, if there was one */ int *size; /* output - how many bytes there are */ { struct message *msg; mblk_t *bp; mblk_t *deque(); int s; ASSERT(rd); /* need to raise priority because arrmsg() calls enque() and add_to_msgs_list() at stream priority */ s = splrf(); if ((bp = deque (&rd->rd_rcvdq)) == NULL) { (void) splx(s); DUPRINT3(DB_COMM, "dequeue: que %x empty,act_cnt=%d\n", rd,rd->rd_act_cnt); return (RCVQEMP); } ASSERT(rd->rd_qcnt > 0); DUPRINT5(DB_COMM, "dequeue: rd_act_cnt=%d que=%x, msg %x qcnt %x\n", rd->rd_act_cnt,rd,bp,rd->rd_qcnt); rd->rd_qcnt--; msg = (struct message *)bp->b_rptr; if ((rcvdemp(rd)) && (rd->rd_qtype & GENERAL)) rm_msgs_list(rd); (void) splx(s); *bufp = bp; *size = msg->m_size - sizeof (struct message); if (msg->m_stat & GIFT) { struct request *msg_in; ASSERT(sd); msg_in = (struct request *) PTOMSG (msg); set_sndd (sd, (queue_t *)msg->m_queue, (index_t) msg->m_gindex, (int) msg->m_gconnid); sd->sd_mntindx = msg_in->rq_mntindx; } return (RFS_SUCCESS); } /* Initialize the communications data structures. */ comminit () { extern rcvd_t sigrd; register sndd_t tmp; rcvd_t rd; register mblk_t *bp; register int i; if (nsndd < MINSNDD || nrcvd < MINRCVD || maxgdp < MINGDP) return(RFS_FAILURE); if (nsndd <= maxserve) { maxserve = (nsndd - 1); printf ("maxserve changed to %d - not enough send descriptors\n", maxserve); } if (maxserve < minserve) { minserve = maxserve; printf ("minserve changed to %d (maxserve) \n", minserve); } for (tmp = sndd ; tmp < &sndd[nsndd]; tmp++) { tmp->sd_stat = SDUNUSED; tmp->sd_next = tmp + 1; } sndd[nsndd - 1].sd_next = NULL; sdfreelist = sndd; if ((sdlowmark = nsndd/10) == 0) sdlowmark = 1; for (rd = rcvd; rd < &rcvd[nrcvd]; rd++) { rd->rd_stat = RDUNUSED; rd->rd_connid = 0; rd->rd_next = rd + 1; } rcvd[nrcvd - 1].rd_next = NULL; rdfreelist = rcvd; rdfree = nrcvd; sdfree = nsndd; /* create well-known RDs & SDs */ cfrd = cr_rcvd(NSQSIZE, GENERAL); /* chow fun */ sigrd = cr_rcvd(SIGQSIZE, GENERAL); /* signals */ rd_recover = cr_rcvd(NSQSIZE, SPECIFIC); /* recovery */ cache_rd = cr_rcvd (1, SPECIFIC); /* cache RD */ cache_sd = cr_sndd (); /* cache SD */ connid = 0; /* *Allocate one buffer for client */ if ((bp = allocb (sizeof(struct message) + sizeof(struct response), BPRI_MED)) == NULL) { printf("WARNING: not enough stream buffers for RFS, RFS failed\n"); u.u_error = ENOMEM; commdinit(); return(RFS_FAILURE); } client_bp = bp; /* allocate maxserve number of 2K stream buffers for server usage */ for (i = 0; i < nserverbuf; i++) { if ((bp = allocb(sizeof(struct message) + sizeof(struct response), BPRI_MED)) == NULL) { /* fail to get enough stream buffers for server usage, fail the RFS startup and free stream buffers */ printf("WARNING: not enough stream buffers for RFS, RFS failed\n"); u.u_error = ENOMEM; commdinit(); return(RFS_FAILURE); } bp->b_wptr += sizeof(struct message); bzero((char *)bp->b_rptr, sizeof(struct message)+ sizeof(struct response)-DATASIZE); bp->b_next = server_bp; server_bp = bp; } return(RFS_SUCCESS); } /* De-initialize the communications data structures. */ void commdinit() { register mblk_t *bp; DUPRINT1 (DB_RFSTART, "commdinit \n"); (void) del_rcvd (cfrd, (struct serv_proc *) NULL); (void) del_rcvd (sigrd, (struct serv_proc *) NULL); (void) del_rcvd (rd_recover, (struct serv_proc *) NULL); (void) del_rcvd (cache_rd, (struct serv_proc *) NULL); free_sndd(cache_sd); while (bp = server_bp) { server_bp = bp->b_next; freemsg(bp); } if (client_bp) { freemsg(client_bp); client_bp = NULL; } } arrmsg (bp) mblk_t *bp; { register struct message *msgp = (struct message *)bp->b_rptr; register rcvd_t rd; extern rcvd_t sigrd; struct serv_proc *found; extern int msgflag; /* * put the message in the right receive queue */ ASSERT (msgp->m_dest >= 0 && msgp->m_dest < nrcvd); rd = inxtord (msgp->m_dest); DUPRINT3(DB_COMM, "arrmsg: msg %x for rd %x\n", bp, rd); /* ASSERT (rd->rd_connid == msgp->m_connid); */ if (msgp->m_stat & SIGNAL) { if(rd->rd_stat == RDUNUSED){ freemsg(bp); DUPRINT2(DB_SIGNAL,"arrmsg:SIG for bad rd %x ignored\n", rd); return; } DUPRINT3(DB_SIGNAL,"arrmsg:m_stat == SIGNAL rd %x sigrd %x\n", rd,sigrd); rd = sigrd; ASSERT (rd->rd_stat != RDUNUSED); } if (rd->rd_stat == RDUNUSED) { printf("RFS: got request for inactive rd %d, ignored\n", msgp->m_dest); freemsg(bp); return; } enque(&rd->rd_rcvdq, bp); rd->rd_qcnt++; if (rd->rd_qtype & SPECIFIC) cvsema((caddr_t) &rd->rd_qslp); else { /*dispatch the first free process*/ if (found = get_proc(&s_idle)) vsema((caddr_t) &found->s_sleep, 0); add_to_msgs_list(rd); if (!found) if(nservers < maxserve){ msgflag |= MORE_SERVE; wakeup((caddr_t) &rd_recover->rd_qslp); /* rfdaemon nacks if no servers and no streams bufs */ } else if ( nservers == maxserve && testb(2048,BPRI_MED) == 0){ msgflag |= DEADLOCK; wakeup((caddr_t) &rd_recover->rd_qslp); } } }