1919#include <linux/stringify.h>
2020#include <linux/bug.h>
2121#include <linux/sched.h>
22+ #include <linux/slab.h>
23+ #include <linux/hashtable.h>
2224
2325/*
2426 * Per-cpu variables require a unique name although static in some
@@ -57,6 +59,9 @@ static struct rv_monitor rv_this;
5759
5860/*
5961 * Type for the target id, default to int but can be overridden.
62+ * A long type can work as hash table key (PER_OBJ) but will be downgraded to
63+ * int in the event tracepoint.
64+ * Unused for implicit monitors.
6065 */
6166#ifndef da_id_type
6267#define da_id_type int
@@ -245,9 +250,9 @@ static inline struct da_monitor *da_get_monitor(struct task_struct *tsk)
245250}
246251
247252/*
248- * da_get_task - return the task associated to the monitor
253+ * da_get_target - return the task associated to the monitor
249254 */
250- static inline struct task_struct * da_get_task (struct da_monitor * da_mon )
255+ static inline struct task_struct * da_get_target (struct da_monitor * da_mon )
251256{
252257 return container_of (da_mon , struct task_struct , rv [task_mon_slot ].da_mon );
253258}
@@ -259,7 +264,7 @@ static inline struct task_struct *da_get_task(struct da_monitor *da_mon)
259264 */
260265static inline da_id_type da_get_id (struct da_monitor * da_mon )
261266{
262- return da_get_task (da_mon )-> pid ;
267+ return da_get_target (da_mon )-> pid ;
263268}
264269
265270static void da_monitor_reset_all (void )
@@ -309,6 +314,221 @@ static inline void da_monitor_destroy(void)
309314
310315 da_monitor_reset_all ();
311316}
317+
318+ #elif RV_MON_TYPE == RV_MON_PER_OBJ
319+ /*
320+ * Functions to define, init and get a per-object monitor.
321+ */
322+
323+ struct da_monitor_storage {
324+ da_id_type id ;
325+ monitor_target target ;
326+ union rv_task_monitor rv ;
327+ struct hlist_node node ;
328+ struct rcu_head rcu ;
329+ };
330+
331+ #ifndef DA_MONITOR_HT_BITS
332+ #define DA_MONITOR_HT_BITS 10
333+ #endif
334+ static DEFINE_HASHTABLE (da_monitor_ht , DA_MONITOR_HT_BITS ) ;
335+
336+ /*
337+ * da_create_empty_storage - pre-allocate an empty storage
338+ */
339+ static inline struct da_monitor_storage * da_create_empty_storage (da_id_type id )
340+ {
341+ struct da_monitor_storage * mon_storage ;
342+
343+ mon_storage = kmalloc_nolock (sizeof (struct da_monitor_storage ),
344+ __GFP_ZERO , NUMA_NO_NODE );
345+ if (!mon_storage )
346+ return NULL ;
347+
348+ hash_add_rcu (da_monitor_ht , & mon_storage -> node , id );
349+ mon_storage -> id = id ;
350+ return mon_storage ;
351+ }
352+
353+ /*
354+ * da_create_storage - create the per-object storage
355+ *
356+ * The caller is responsible to synchronise writers, either with locks or
357+ * implicitly. For instance, if da_create_storage is only called from a single
358+ * event for target (e.g. sched_switch), it's safe to call this without locks.
359+ */
360+ static inline struct da_monitor * da_create_storage (da_id_type id ,
361+ monitor_target target ,
362+ struct da_monitor * da_mon )
363+ {
364+ struct da_monitor_storage * mon_storage ;
365+
366+ if (da_mon )
367+ return da_mon ;
368+
369+ mon_storage = da_create_empty_storage (id );
370+ if (!mon_storage )
371+ return NULL ;
372+
373+ mon_storage -> target = target ;
374+ return & mon_storage -> rv .da_mon ;
375+ }
376+
377+ /*
378+ * __da_get_mon_storage - get the monitor storage from the hash table
379+ */
380+ static inline struct da_monitor_storage * __da_get_mon_storage (da_id_type id )
381+ {
382+ struct da_monitor_storage * mon_storage ;
383+
384+ lockdep_assert_in_rcu_read_lock ();
385+ hash_for_each_possible_rcu (da_monitor_ht , mon_storage , node , id ) {
386+ if (mon_storage -> id == id )
387+ return mon_storage ;
388+ }
389+
390+ return NULL ;
391+ }
392+
393+ /*
394+ * da_get_monitor - return the monitor for target
395+ */
396+ static struct da_monitor * da_get_monitor (da_id_type id , monitor_target target )
397+ {
398+ struct da_monitor_storage * mon_storage ;
399+
400+ mon_storage = __da_get_mon_storage (id );
401+ return mon_storage ? & mon_storage -> rv .da_mon : NULL ;
402+ }
403+
404+ /*
405+ * da_get_target - return the object associated to the monitor
406+ */
407+ static inline monitor_target da_get_target (struct da_monitor * da_mon )
408+ {
409+ return container_of (da_mon , struct da_monitor_storage , rv .da_mon )-> target ;
410+ }
411+
412+ /*
413+ * da_get_id - return the id associated to the monitor
414+ */
415+ static inline da_id_type da_get_id (struct da_monitor * da_mon )
416+ {
417+ return container_of (da_mon , struct da_monitor_storage , rv .da_mon )-> id ;
418+ }
419+
420+ /*
421+ * da_create_or_get - create the per-object storage if not already there
422+ *
423+ * This needs a lookup so should be guarded by RCU, the condition is checked
424+ * directly in da_create_storage()
425+ */
426+ static inline void da_create_or_get (da_id_type id , monitor_target target )
427+ {
428+ guard (rcu )();
429+ da_create_storage (id , target , da_get_monitor (id , target ));
430+ }
431+
432+ /*
433+ * da_fill_empty_storage - store the target in a pre-allocated storage
434+ *
435+ * Can be used as a substitute of da_create_storage when starting a monitor in
436+ * an environment where allocation is unsafe.
437+ */
438+ static inline struct da_monitor * da_fill_empty_storage (da_id_type id ,
439+ monitor_target target ,
440+ struct da_monitor * da_mon )
441+ {
442+ if (unlikely (da_mon && !da_get_target (da_mon )))
443+ container_of (da_mon , struct da_monitor_storage , rv .da_mon )-> target = target ;
444+ return da_mon ;
445+ }
446+
447+ /*
448+ * da_get_target_by_id - return the object associated to the id
449+ */
450+ static inline monitor_target da_get_target_by_id (da_id_type id )
451+ {
452+ struct da_monitor_storage * mon_storage ;
453+
454+ guard (rcu )();
455+ mon_storage = __da_get_mon_storage (id );
456+
457+ if (unlikely (!mon_storage ))
458+ return NULL ;
459+ return mon_storage -> target ;
460+ }
461+
462+ /*
463+ * da_destroy_storage - destroy the per-object storage
464+ *
465+ * The caller is responsible to synchronise writers, either with locks or
466+ * implicitly. For instance, if da_destroy_storage is called at sched_exit and
467+ * da_create_storage can never occur after that, it's safe to call this without
468+ * locks.
469+ * This function includes an RCU read-side critical section to synchronise
470+ * against da_monitor_destroy().
471+ */
472+ static inline void da_destroy_storage (da_id_type id )
473+ {
474+ struct da_monitor_storage * mon_storage ;
475+
476+ guard (rcu )();
477+ mon_storage = __da_get_mon_storage (id );
478+
479+ if (!mon_storage )
480+ return ;
481+ da_monitor_reset_hook (& mon_storage -> rv .da_mon );
482+ hash_del_rcu (& mon_storage -> node );
483+ kfree_rcu (mon_storage , rcu );
484+ }
485+
486+ static void da_monitor_reset_all (void )
487+ {
488+ struct da_monitor_storage * mon_storage ;
489+ int bkt ;
490+
491+ rcu_read_lock ();
492+ hash_for_each_rcu (da_monitor_ht , bkt , mon_storage , node )
493+ da_monitor_reset (& mon_storage -> rv .da_mon );
494+ rcu_read_unlock ();
495+ }
496+
497+ static inline int da_monitor_init (void )
498+ {
499+ hash_init (da_monitor_ht );
500+ return 0 ;
501+ }
502+
503+ static inline void da_monitor_destroy (void )
504+ {
505+ struct da_monitor_storage * mon_storage ;
506+ struct hlist_node * tmp ;
507+ int bkt ;
508+
509+ /*
510+ * This function is called after all probes are disabled, we need only
511+ * worry about concurrency against old events.
512+ */
513+ synchronize_rcu ();
514+ hash_for_each_safe (da_monitor_ht , bkt , tmp , mon_storage , node ) {
515+ da_monitor_reset_hook (& mon_storage -> rv .da_mon );
516+ hash_del_rcu (& mon_storage -> node );
517+ kfree (mon_storage );
518+ }
519+ }
520+
521+ /*
522+ * Allow the per-object monitors to run allocation manually, necessary if the
523+ * start condition is in a context problematic for allocation (e.g. scheduling).
524+ * In such case, if the storage was pre-allocated without a target, set it now.
525+ */
526+ #ifdef DA_SKIP_AUTO_ALLOC
527+ #define da_prepare_storage da_fill_empty_storage
528+ #else
529+ #define da_prepare_storage da_create_storage
530+ #endif /* DA_SKIP_AUTO_ALLOC */
531+
312532#endif /* RV_MON_TYPE */
313533
314534#if RV_MON_TYPE == RV_MON_GLOBAL || RV_MON_TYPE == RV_MON_PER_CPU
@@ -342,9 +562,9 @@ static inline da_id_type da_get_id(struct da_monitor *da_mon)
342562 return 0 ;
343563}
344564
345- #elif RV_MON_TYPE == RV_MON_PER_TASK
565+ #elif RV_MON_TYPE == RV_MON_PER_TASK || RV_MON_TYPE == RV_MON_PER_OBJ
346566/*
347- * Trace events for per_task monitors, report the PID of the task .
567+ * Trace events for per_task/per_object monitors, report the target id .
348568 */
349569
350570static inline void da_trace_event (struct da_monitor * da_mon ,
@@ -525,6 +745,76 @@ static inline bool da_handle_start_run_event(struct task_struct *tsk,
525745{
526746 return __da_handle_start_run_event (da_get_monitor (tsk ), event , tsk -> pid );
527747}
748+
749+ #elif RV_MON_TYPE == RV_MON_PER_OBJ
750+ /*
751+ * Handle event for per object.
752+ */
753+
754+ /*
755+ * da_handle_event - handle an event
756+ */
757+ static inline void da_handle_event (da_id_type id , monitor_target target , enum events event )
758+ {
759+ struct da_monitor * da_mon ;
760+
761+ guard (rcu )();
762+ da_mon = da_get_monitor (id , target );
763+ if (likely (da_mon ))
764+ __da_handle_event (da_mon , event , id );
765+ }
766+
767+ /*
768+ * da_handle_start_event - start monitoring or handle event
769+ *
770+ * This function is used to notify the monitor that the system is returning
771+ * to the initial state, so the monitor can start monitoring in the next event.
772+ * Thus:
773+ *
774+ * If the monitor already started, handle the event.
775+ * If the monitor did not start yet, start the monitor but skip the event.
776+ */
777+ static inline bool da_handle_start_event (da_id_type id , monitor_target target ,
778+ enum events event )
779+ {
780+ struct da_monitor * da_mon ;
781+
782+ guard (rcu )();
783+ da_mon = da_get_monitor (id , target );
784+ da_mon = da_prepare_storage (id , target , da_mon );
785+ if (unlikely (!da_mon ))
786+ return 0 ;
787+ return __da_handle_start_event (da_mon , event , id );
788+ }
789+
790+ /*
791+ * da_handle_start_run_event - start monitoring and handle event
792+ *
793+ * This function is used to notify the monitor that the system is in the
794+ * initial state, so the monitor can start monitoring and handling event.
795+ */
796+ static inline bool da_handle_start_run_event (da_id_type id , monitor_target target ,
797+ enum events event )
798+ {
799+ struct da_monitor * da_mon ;
800+
801+ guard (rcu )();
802+ da_mon = da_get_monitor (id , target );
803+ da_mon = da_prepare_storage (id , target , da_mon );
804+ if (unlikely (!da_mon ))
805+ return 0 ;
806+ return __da_handle_start_run_event (da_mon , event , id );
807+ }
808+
809+ static inline void da_reset (da_id_type id , monitor_target target )
810+ {
811+ struct da_monitor * da_mon ;
812+
813+ guard (rcu )();
814+ da_mon = da_get_monitor (id , target );
815+ if (likely (da_mon ))
816+ da_monitor_reset (da_mon );
817+ }
528818#endif /* RV_MON_TYPE */
529819
530820#endif
0 commit comments