Do not shelve non-class subgoals but fail, it should

be the instance writer's responsibility to not generated non-dependent
non-class subgoals (otherwise we loose compatibility as shown in
e.g. MathClasses, which goes into loops because of unexpectedly
unconstrained goals). Reflect it in the doc.

2 files changed, 12 insertions, 9 deletions

diff --git a/doc/refman/Classes.tex b/doc/refman/Classes.tex
index d6a553e1a8..58ae7191f8 100644
--- a/doc/refman/Classes.tex
+++ b/doc/refman/Classes.tex
@@ -392,11 +392,13 @@ than {\tt eauto} and {\tt auto}. The main differences are the following:
   backtracking on subgoals that are entirely independent.
 \item When called with no arguments, {\tt typeclasses eauto} uses the
   {\tt typeclass\_instances} database by default (instead of {\tt core})
-  and will try to solve \emph{only} typeclass goals. Other subgoals are
-  automatically shelved and \emph{must be} resolved entirely when the
-  other typeclass subgoals are resolved or the proof search will fail
-  \emph{globally}, \emph{without} the possibility to find another
-  complete solution with no shelved subgoals.
+  and will try to solve \emph{only} typeclass goals. If some subgoal of
+  a hint/instance is non-dependent and not of class type, that hint
+  application will fail. Dependent subgoals are automatically shelved
+  and \emph{must be} resolved entirely when the other typeclass subgoals
+  are resolved or the proof search will fail \emph{globally},
+  \emph{without} the possibility to find another complete solution with
+  no shelved subgoals.
 
   \emph{Note: } As of Coq 8.6, {\tt all:once (typeclasses eauto)}
   faithfully mimicks what happens during typeclass resolution when it is
diff --git a/tactics/class_tactics.ml b/tactics/class_tactics.ml
index 91eb388b3e..fe12b54a13 100644
--- a/tactics/class_tactics.ml
+++ b/tactics/class_tactics.ml
@@ -1020,16 +1020,17 @@ module Search = struct
 	Evd.add sigma gl evi')
       sigma goals
 
-  let shelve_nonclass info =
+  let fail_if_nonclass info =
     Proofview.Goal.enter { enter = fun gl ->
       let gl = Proofview.Goal.assume gl in
       let sigma = Sigma.to_evar_map (Proofview.Goal.sigma gl) in
       if is_class_type sigma (Proofview.Goal.concl gl) then
         Proofview.tclUNIT ()
       else (if !typeclasses_debug > 1 then
-              Feedback.msg_debug (pr_depth info.search_depth ++ str": shelving non-class subgoal " ++
+              Feedback.msg_debug (pr_depth info.search_depth ++
+                                    str": failure due to non-class subgoal " ++
                                     pr_ev sigma (Proofview.Goal.goal gl));
-            Proofview.shelve) }
+            Proofview.tclZERO NotApplicableEx) }
 
   (** The general hint application tactic.
       tac1 + tac2 .... The choice of OR or ORELSE is determined
@@ -1159,7 +1160,7 @@ module Search = struct
       if path_matches derivs [] then aux e tl
       else
         let filter =
-          if info.search_only_classes then shelve_nonclass info
+          if info.search_only_classes then fail_if_nonclass info
           else Proofview.tclUNIT ()
         in
         ortac