As mentioned in Specifying Redirection Functions, achieving acceptable optimality of emitting output signals by an actor may require it to keep statistics for a tail of the event history rather than for the entire event history. If an actor learns a state model, keeping statistics for a tail of the event history makes a current state model less stable allowing the actor to try more state model variations. After trying state model variations with counting frequencies of state transitions performed, an application program can partially determinize a current state model by disallowing some less frequent state transitions and continue state model training. A state model fully determinized at the end of iterative partial determinization becomes a learned deterministic state model.
The file samples/cyc_stats_upd_hook.c in the package distribution installed to $prefix/share/qsmm/samples/cyc_stats_upd_hook.c contains:
cycle_stats_update_hook for intercepting updates of statistics on a cycle type to implement keeping cycle type statistics for an event history tail;
relprob_user2 for calculating the relative probability of an output signal based on cycle type statistics for an event history tail;
setup_actor setting the functions cycle_stats_update_hook and relprob_user2 for a large actor.
All functions use an allocated instance of hook_param_s structure initially zeroed and passed to setup_actor.
This section includes the source code of the three functions and associated macros and data structures.
To organize storing statistics on cycle types for a tail of the event history of a small actor associated with the large actor, cycle_stats_update_hook keeps a queue of records held in the instances of histev_cycle_s structure, where every record contains the following information about an update of statistics on a cycle type:
stepvirtual time of the update designated as “modeling step”
spurspur accumulated by the actor
tmccontinuous time of the update
sig_ngramsignal array encoding an action choice state that along with a cycle direction sig_cycle specifies the cycle type
sig_cyclecycle direction that along with an action choice state encoded by sig_ngram specifies the cycle type
fq_deltaincrement to the field fq of qsmm_cycle_s structure equal to a difference between a new value and old value passed to the redirection function
period_sum_d_deltaincrement to the field period_sum_d of qsmm_cycle_s structure equal to a difference between a new value and old value passed to the redirection function
period_sum_c_deltaincrements to continuous time in the field delta_sum in the elements of array of qsmm_cspval_s structures for continuous time types equal to differences between new values and old values passed to the redirection function
spur_delta_sumincrements to spur in the field delta_sum in the elements of array of qsmm_cspval_s structures for spur types equal to differences between new values and old values passed to the redirection function
The redirection function does the following:
qsmm_set_storage_cycle_stats to decrement statistics on cycle types.
An application program should keep the fields step_histev and nstep_histev in the instance of hook_param_s structure up-to-date while performing state model training.
The field step_histev holds a modeling step index equal, for example, to the number of nondeterministic state transitions performed.
The field nstep_histev holds the length of an event history tail in modeling steps.
Calculating the length of an event history tail to keep statistics is out of scope of this section.
#include <assert.h>
#include <math.h>
#include <stdlib.h>
#include <string.h>
#include <qsmm/qsmm.h>
#define HISTEV_CYCLE_NSPUR 2 // number of spur types
#define HISTEV_CYCLE_NTIME 2 // number of continuous time types
#define HISTEV_CYCLE_NGRAM_SZ 4
// length of action choice state n-grams of a small actor associated
// with a large actor
#define GET_NSTEP_HISTEV(hook_param_p) \
((long) ((hook_param_p)->nstep_histev+0.5))
#define ERREXIT(fmt, ...) \
do { \
fprintf(stderr,fmt "\n", ## __VA_ARGS__); \
goto Exit; \
} \
while (0)
#define ERR_NOMEM() ERREXIT("out of memory")
#define CHK_FAIL(func, ...) \
do { \
const int rc=func(__VA_ARGS__); \
if (rc<0) ERREXIT(#func ": %s", qsmm_err_str(rc)); \
} \
while (0)
struct histev_cycle_s {
long step;
// modeling step
long fq_delta;
// increment of cycle type frequency
long period_sum_d_delta;
// increment of the sum of discrete time periods
double spur[HISTEV_CYCLE_NSPUR];
// actor spur:
// [ii] = spur with type `ii'
double tmc[HISTEV_CYCLE_NTIME];
// continuous time:
// [ii] = continuous time with type `ii'
double period_sum_c_delta[HISTEV_CYCLE_NTIME];
// increments of the sums of continuous time periods:
// [ii] = increment of the sum of periods of continuous
// time with type `ii'
double spur_delta_sum[HISTEV_CYCLE_NSPUR];
// increments of the sums of spur increments:
// [ii] = increment of the sum of spur increments for
// spur type `ii'
qsmm_sig_t sig_cycle;
// cycle direction
qsmm_sig_t sig_ngram[HISTEV_CYCLE_NGRAM_SZ];
// action choice state n-gram
struct histev_cycle_s *nextp;
// next event in the list;
// special value:
// ==NULL - this event is the last event in the list
};
struct hook_param_s {
char in_cycle_update_hook;
// cycle update hook execution indicator:
// !=0 - cycle statistics update hook is being executed;
// ==0 - not executed
long step_histev;
// modeling step
double nstep_histev;
// number of modeling steps within the cycle event history window
qsmm_actor_t actor_small;
// reference to a small actor corresponding to the environment
// state identification engine
struct histev_cycle_s *histev_cycle_head_p;
// oldest event in the history of last cycle events;
// special value:
// ==NULL - history is empty
struct histev_cycle_s *histev_cycle_tail_p;
// newest event in the history of last cycle events;
// special value:
// ==NULL - history is empty
};
// The hook for calling on updating cycle type statistics.
// Returns: 0 = success; no return on failure.
static int
cycle_stats_update_hook(
qsmm_storage_t storage,
int ngram_sz,
const qsmm_sig_t *sig_ngram_p, // [ngram_sz]
qsmm_sig_t sig_cycle,
const struct qsmm_cycle_s *cycle_new_p,
struct qsmm_cycle_s *cycle_result_p,
const struct qsmm_cspval_s *cspval_new_p, // [nspval]
struct qsmm_cspval_s *cspval_result_p, // [nspval]
void *paramp
) {
int result=-1;
struct histev_cycle_s *histev_new_p=0;
struct hook_param_s *const hook_param_p=paramp;
if (hook_param_p->in_cycle_update_hook) {
result=0;
goto Exit;
}
hook_param_p->in_cycle_update_hook=1;
const long step_beg=
hook_param_p->step_histev-GET_NSTEP_HISTEV(hook_param_p);
const qsmm_actor_t actor_small=hook_param_p->actor_small;
const int nspur=qsmm_get_actor_nspur(actor_small),
ntime=qsmm_get_actor_ntime(actor_small),
nspval=qsmm_get_storage_nspval(storage);
int ispur, itime;
struct histev_cycle_s *histev_old_p;
assert(ngram_sz==HISTEV_CYCLE_NGRAM_SZ);
assert(nspur==HISTEV_CYCLE_NSPUR);
assert(ntime==HISTEV_CYCLE_NTIME);
assert(nspur+ntime==nspval);
while ((histev_old_p=hook_param_p->histev_cycle_head_p) &&
histev_old_p->step<step_beg) {
const double *const spur_delta_sum_p=histev_old_p->spur_delta_sum;
const qsmm_sig_t sig_cycle_old=histev_old_p->sig_cycle,
*const sig_ngram_old_p=histev_old_p->sig_ngram;
struct qsmm_cycle_s cycle;
struct qsmm_cspval_s cspval[nspval];
CHK_FAIL(qsmm_get_storage_cycle_stats, storage, ngram_sz,
sig_ngram_old_p, sig_cycle_old, &cycle, cspval);
cycle.fq-=histev_old_p->fq_delta;
cycle.period_sum_d-=histev_old_p->period_sum_d_delta;
assert(cycle.fq>=0);
assert(cycle.period_sum_d>=0);
for (ispur=0; ispur<nspur; ispur++)
cspval[ispur].delta_sum-=spur_delta_sum_p[ispur];
for (itime=0; itime<ntime; itime++) {
cspval[nspur+itime].delta_sum-=
histev_old_p->period_sum_c_delta[itime];
assert(cspval[nspur+itime].delta_sum>=0);
}
CHK_FAIL(qsmm_set_storage_cycle_stats, storage, ngram_sz,
sig_ngram_old_p, sig_cycle_old, &cycle, cspval);
hook_param_p->histev_cycle_head_p=histev_old_p->nextp;
free(histev_old_p);
}
if (!histev_old_p) hook_param_p->histev_cycle_tail_p=0;
if (!(histev_new_p=calloc(1,sizeof(*histev_new_p)))) ERR_NOMEM();
histev_new_p->step=hook_param_p->step_histev;
histev_new_p->sig_cycle=sig_cycle;
if (cycle_new_p) {
histev_new_p->fq_delta=cycle_new_p->fq-cycle_result_p->fq;
histev_new_p->period_sum_d_delta=cycle_new_p->period_sum_d-
cycle_result_p->period_sum_d;
assert(histev_new_p->fq_delta>=0);
assert(histev_new_p->period_sum_d_delta>=0);
}
for (ispur=0; ispur<nspur; ispur++) {
CHK_FAIL(qsmm_get_actor_spur, actor_small, ispur,
histev_new_p->spur+ispur);
histev_new_p->spur_delta_sum[ispur]=
cspval_new_p?cspval_new_p[ispur].delta_sum-
cspval_result_p[ispur].delta_sum:0;
}
for (itime=0; itime<ntime; itime++) {
CHK_FAIL(qsmm_get_actor_time,
actor_small, itime, histev_new_p->tmc+itime);
histev_new_p->period_sum_c_delta[itime]=
cspval_new_p?cspval_new_p[nspur+itime].delta_sum-
cspval_result_p[nspur+itime].delta_sum:0;
assert(histev_new_p->period_sum_c_delta[itime]>=0);
}
memmove(histev_new_p->sig_ngram, sig_ngram_p,
ngram_sz*sizeof(*sig_ngram_p));
if (hook_param_p->histev_cycle_tail_p) {
assert(!hook_param_p->histev_cycle_tail_p->nextp);
hook_param_p->histev_cycle_tail_p->nextp=histev_new_p;
}
else {
assert(!hook_param_p->histev_cycle_head_p);
hook_param_p->histev_cycle_head_p=histev_new_p;
}
hook_param_p->histev_cycle_tail_p=histev_new_p;
histev_new_p=0;
hook_param_p->in_cycle_update_hook=0;
result=0;
Exit:
if (histev_new_p) free(histev_new_p);
if (result<0) exit(1);
return result;
}
// The helper function for computing the relative probability of an
// output signal.
static double
relprob_user2(
qsmm_actor_t actor,
qsmm_sig_t sig_cycle,
const struct qsmm_state_s *state_p,
const struct qsmm_cycle_s *cycle_p,
const struct qsmm_sspval_s *sspval_p,
const struct qsmm_cspval_s *cspval_p,
void *paramp
) {
const long fq=cycle_p->fq;
double result=0/0.0;
if (fq<1) {
result=0;
goto Exit;
}
const struct histev_cycle_s *const histev_head_p=
((struct hook_param_s *) paramp)->histev_cycle_head_p;
const int nspur=qsmm_get_actor_nspur(actor);
double exp_arg=0;
assert(histev_head_p);
for (int ispur=0; ispur<nspur; ispur++) {
int itime=-1;
CHK_FAIL(qsmm_get_actor_spur_time,actor,ispur,&itime);
assert(itime>=0);
double spur_val=0, time_val=0;
CHK_FAIL(qsmm_get_actor_spur,actor,ispur,&spur_val);
CHK_FAIL(qsmm_get_actor_time,actor,itime,&time_val);
spur_val-=histev_head_p->spur[ispur];
time_val-=histev_head_p->tmc[itime];
const double spur_delta_mean=time_val>0?spur_val/time_val:0;
double spur_weight=0;
CHK_FAIL(qsmm_get_actor_spur_weight,actor,ispur,&spur_weight);
const double num=cspval_p[ispur].delta_sum,
den=cspval_p[nspur+itime].delta_sum*fabs(spur_delta_mean);
enum qsmm_spur_perception_e spur_perception;
CHK_FAIL(qsmm_get_actor_spur_perception, actor,
ispur, &spur_perception);
switch (spur_perception) {
case QSMM_SPUR_PERCEPTION_NORMAL:
if (den) exp_arg+=spur_weight*num/den;
break;
case QSMM_SPUR_PERCEPTION_INVERSE:
if (num) exp_arg+=-spur_weight*den/num;
break;
}
}
assert(cycle_p->period_sum_d>0);
exp_arg*=log(qsmm_get_actor_nsig_ctrl(actor))*cycle_p->period_sum_d/fq;
result=exp_arg;
Exit:
return result;
}
// Set up a hook for updating cycle type statistics and a helper function
// for computing the relative probability of an output signal for a
// large actor.
// Returns: 0 = success;
// -1 = failure.
int
setup_actor(
qsmm_actor_t actor_large,
struct hook_param_s *hook_param_p
) {
int result=-1;
qsmm_set_actor_relprob_type(actor_large,QSMM_RELPROB_USER2);
qsmm_set_actor_relprob_helper(actor_large,&relprob_user2,hook_param_p);
const qsmm_actor_t actor_small=
qsmm_get_actpair_actor(
qsmm_get_actpair(qsmm_get_actor_large_model(actor_large)),
QSMM_ENGINE_ENV);
CHK_FAIL(qsmm_set_storage_cycle_update_hook,
qsmm_get_actor_storage(actor_small),
&cycle_stats_update_hook, hook_param_p);
hook_param_p->actor_small=actor_small;
hook_param_p->step_histev=0;
result=0;
Exit:
return result;
}